php/archived-php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2026-03-24 00:02:20 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
097edc86c820178da7b26c50bb87693b45e3be8e
archived-php-src/ext/opcache/jit/zend_jit_ir.c
Dmitry Stogov 097edc86c8 Allow JIT for passing arguments to trampolines and "bad" functions (#16365) 
* Better trace coverage (JIT trampoline calls)

* clenup trampoline by zend_jit_free_trampoline()

* Fix ZEND_JIT_TRACE_INIT_CALL/ZEND_JIT_TRACE_DO_ICALL num_args mismatch

It may be caused by SEND_UNPACK/SEND_ARRAY

* cleanup

* cleanup

* Don't record function that may be temporary

* cleanup

* Prevent invalid run_time_cache allocation for "bad" internal functions

* Update zend_jit_trace_record_fake_init_call_ex() accordingly

* Better handling of "bad" functions and fake closures
2024-10-15 22:31:05 +03:00

17758 lines
559 KiB
C
Raw Blame History

/*
* +----------------------------------------------------------------------+
* | Zend JIT |
* +----------------------------------------------------------------------+
* | Copyright (c) The PHP Group |
* +----------------------------------------------------------------------+
* | This source file is subject to version 3.01 of the PHP license, |
* | that is bundled with this package in the file LICENSE, and is |
* | available through the world-wide-web at the following url: |
* | https://www.php.net/license/3_01.txt |
* | If you did not receive a copy of the PHP license and are unable to |
* | obtain it through the world-wide-web, please send a note to |
* | license@php.net so we can mail you a copy immediately. |
* +----------------------------------------------------------------------+
* | Authors: Dmitry Stogov <dmitry@php.net> |
* +----------------------------------------------------------------------+
*/
#include "jit/ir/ir.h"
#include "jit/ir/ir_builder.h"
#if defined(IR_TARGET_X86)
# define IR_REG_SP 4 /* IR_REG_RSP */
# define IR_REG_FP 5 /* IR_REG_RBP */
# define ZREG_FP 6 /* IR_REG_RSI */
# define ZREG_IP 7 /* IR_REG_RDI */
# define ZREG_FIRST_FPR 8
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */
#elif defined(IR_TARGET_X64)
# define IR_REG_SP 4 /* IR_REG_RSP */
# define IR_REG_FP 5 /* IR_REG_RBP */
# define ZREG_FP 14 /* IR_REG_R14 */
# define ZREG_IP 15 /* IR_REG_R15 */
# define ZREG_FIRST_FPR 16
# if defined(_WIN64)
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15))
/*
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15) | \
(1<<(16+6)) | (1<<(16+7)) | (1<<(16+8)) | (1<<(16+9)) | (1<<(16+10)) | \
(1<<(16+11)) | (1<<(16+12)) | (1<<(16+13)) | (1<<(16+14)) | (1<<(16+15)))
*/
# else
# define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* all preserved registers */
# endif
#elif defined(IR_TARGET_AARCH64)
# define IR_REG_SP 31 /* IR_REG_RSP */
# define IR_REG_FP 29 /* IR_REG_X29 */
# define ZREG_FP 27 /* IR_REG_X27 */
# define ZREG_IP 28 /* IR_REG_X28 */
# define ZREG_FIRST_FPR 32
# define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \
(1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */
#else
# error "Unknown IR target"
#endif
#define ZREG_RX ZREG_IP
#define OPTIMIZE_FOR_SIZE 0
/* IR builder defines */
#undef _ir_CTX
#define _ir_CTX (&jit->ctx)
#undef ir_CONST_ADDR
#define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr))
#define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0)
#define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC)
#define ir_CAST_FC_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \
ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32))
#define ir_CONST_FUNC_PROTO(_addr, _proto) \
jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto))
#undef ir_ADD_OFFSET
#define ir_ADD_OFFSET(_addr, _offset) \
jit_ADD_OFFSET(jit, _addr, _offset)
#ifdef ZEND_ENABLE_ZVAL_LONG64
# define IR_LONG IR_I64
# define ir_CONST_LONG ir_CONST_I64
# define ir_UNARY_OP_L ir_UNARY_OP_I64
# define ir_BINARY_OP_L ir_BINARY_OP_I64
# define ir_ADD_L ir_ADD_I64
# define ir_SUB_L ir_SUB_I64
# define ir_MUL_L ir_MUL_I64
# define ir_DIV_L ir_DIV_I64
# define ir_MOD_L ir_MOD_I64
# define ir_NEG_L ir_NEG_I64
# define ir_ABS_L ir_ABS_I64
# define ir_SEXT_L ir_SEXT_I64
# define ir_ZEXT_L ir_ZEXT_I64
# define ir_TRUNC_L ir_TRUNC_I64
# define ir_BITCAST_L ir_BITCAST_I64
# define ir_FP2L ir_FP2I64
# define ir_ADD_OV_L ir_ADD_OV_I64
# define ir_SUB_OV_L ir_SUB_OV_I64
# define ir_MUL_OV_L ir_MUL_OV_I64
# define ir_NOT_L ir_NOT_I64
# define ir_OR_L ir_OR_I64
# define ir_AND_L ir_AND_I64
# define ir_XOR_L ir_XOR_I64
# define ir_SHL_L ir_SHL_I64
# define ir_SHR_L ir_SHR_I64
# define ir_SAR_L ir_SAR_I64
# define ir_ROL_L ir_ROL_I64
# define ir_ROR_L ir_ROR_I64
# define ir_MIN_L ir_MIN_I64
# define ir_MAX_L ir_MAX_I64
# define ir_LOAD_L ir_LOAD_I64
#else
# define IR_LONG IR_I32
# define ir_CONST_LONG ir_CONST_I32
# define ir_UNARY_OP_L ir_UNARY_OP_I32
# define ir_BINARY_OP_L ir_BINARY_OP_I32
# define ir_ADD_L ir_ADD_I32
# define ir_SUB_L ir_SUB_I32
# define ir_MUL_L ir_MUL_I32
# define ir_DIV_L ir_DIV_I32
# define ir_MOD_L ir_MOD_I32
# define ir_NEG_L ir_NEG_I32
# define ir_ABS_L ir_ABS_I32
# define ir_SEXT_L ir_SEXT_I32
# define ir_ZEXT_L ir_ZEXT_I32
# define ir_TRUNC_L ir_TRUNC_I32
# define ir_BITCAST_L ir_BITCAST_I32
# define ir_FP2L ir_FP2I32
# define ir_ADD_OV_L ir_ADD_OV_I32
# define ir_SUB_OV_L ir_SUB_OV_I32
# define ir_MUL_OV_L ir_MUL_OV_I32
# define ir_NOT_L ir_NOT_I32
# define ir_OR_L ir_OR_I32
# define ir_AND_L ir_AND_I32
# define ir_XOR_L ir_XOR_I32
# define ir_SHL_L ir_SHL_I32
# define ir_SHR_L ir_SHR_I32
# define ir_SAR_L ir_SAR_I32
# define ir_ROL_L ir_ROL_I32
# define ir_ROR_L ir_ROR_I32
# define ir_MIN_L ir_MIN_I32
# define ir_MAX_L ir_MAX_I32
# define ir_LOAD_L ir_LOAD_I32
#endif
/* A helper structure to collect IT rers for the following use in (MERGE/PHI)_N */
typedef struct _ir_refs {
uint32_t count;
uint32_t limit;
ir_ref refs[] ZEND_ELEMENT_COUNT(count);
} ir_refs;
#define ir_refs_size(_n) (offsetof(ir_refs, refs) + sizeof(ir_ref) * (_n))
#define ir_refs_init(_name, _n) _name = alloca(ir_refs_size(_n)); \
do {_name->count = 0; _name->limit = (_n);} while (0)
static void ir_refs_add(ir_refs *refs, ir_ref ref)
{
ir_ref *ptr;
ZEND_ASSERT(refs->count < refs->limit);
ptr = refs->refs;
ptr[refs->count++] = ref;
}
static size_t zend_jit_trace_prologue_size = (size_t)-1;
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
static uint32_t allowed_opt_flags = 0;
static uint32_t default_mflags = 0;
#endif
static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ???
#ifdef ZTS
# ifdef _WIN32
extern uint32_t _tls_index;
extern char *_tls_start;
extern char *_tls_end;
# endif
static size_t tsrm_ls_cache_tcb_offset = 0;
static size_t tsrm_tls_index = 0;
static size_t tsrm_tls_offset = 0;
# define EG_TLS_OFFSET(field) \
(executor_globals_offset + offsetof(zend_executor_globals, field))
# define CG_TLS_OFFSET(field) \
(compiler_globals_offset + offsetof(zend_compiler_globals, field))
# define jit_EG(_field) \
ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field))
# define jit_CG(_field) \
ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field))
#else
# define jit_EG(_field) \
ir_CONST_ADDR(&EG(_field))
# define jit_CG(_field) \
ir_CONST_ADDR(&CG(_field))
#endif
#define jit_CALL(_call, _field) \
ir_ADD_OFFSET(_call, offsetof(zend_execute_data, _field))
#define jit_EX(_field) \
jit_CALL(jit_FP(jit), _field)
#define jit_RX(_field) \
jit_CALL(jit_IP(jit), _field)
#define JIT_STUBS(_) \
_(exception_handler, IR_SKIP_PROLOGUE) \
_(exception_handler_undef, IR_SKIP_PROLOGUE) \
_(exception_handler_free_op2, IR_SKIP_PROLOGUE) \
_(exception_handler_free_op1_op2, IR_SKIP_PROLOGUE) \
_(interrupt_handler, IR_SKIP_PROLOGUE) \
_(leave_function_handler, IR_SKIP_PROLOGUE) \
_(negative_shift, IR_SKIP_PROLOGUE) \
_(mod_by_zero, IR_SKIP_PROLOGUE) \
_(invalid_this, IR_SKIP_PROLOGUE) \
_(undefined_function, IR_SKIP_PROLOGUE) \
_(throw_cannot_pass_by_ref, IR_SKIP_PROLOGUE) \
_(icall_throw, IR_SKIP_PROLOGUE) \
_(leave_throw, IR_SKIP_PROLOGUE) \
_(hybrid_runtime_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_profile_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_func_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_loop_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_func_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_ret_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(hybrid_loop_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \
_(trace_halt, IR_SKIP_PROLOGUE) \
_(trace_escape, IR_SKIP_PROLOGUE) \
_(trace_exit, IR_SKIP_PROLOGUE) \
_(undefined_offset, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(undefined_key, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(cannot_add_element, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_const, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_tmp, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_var, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_cv_noref, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(assign_cv, IR_FUNCTION | IR_FASTCALL_FUNC) \
_(new_array, IR_FUNCTION | IR_FASTCALL_FUNC) \
#define JIT_STUB_ID(name, flags) \
jit_stub_ ## name,
#define JIT_STUB_FORWARD(name, flags) \
static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit);
#define JIT_STUB(name, flags) \
{JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, flags},
typedef enum _jit_stub_id {
JIT_STUBS(JIT_STUB_ID)
jit_last_stub
} jit_stub_id;
typedef struct _zend_jit_reg_var {
ir_ref ref;
uint32_t flags;
} zend_jit_reg_var;
typedef struct _zend_jit_ctx {
ir_ctx ctx;
const zend_op *last_valid_opline;
bool use_last_valid_opline;
bool track_last_valid_opline;
bool reuse_ip;
uint32_t delayed_call_level;
int b; /* current basic block number or -1 */
#ifdef ZTS
ir_ref tls;
#endif
ir_ref fp;
ir_ref trace_loop_ref;
ir_ref return_inputs;
const zend_op_array *op_array;
const zend_op_array *current_op_array;
zend_ssa *ssa;
zend_string *name;
ir_ref *bb_start_ref; /* PHP BB -> IR ref mapping */
ir_ref *bb_predecessors; /* PHP BB -> index in bb_edges -> IR refs of predessors */
ir_ref *bb_edges;
zend_jit_trace_info *trace;
zend_jit_reg_var *ra;
int delay_var;
ir_refs *delay_refs;
ir_ref eg_exception_addr;
HashTable addr_hash;
ir_ref stub_addr[jit_last_stub];
} zend_jit_ctx;
typedef int8_t zend_reg;
typedef struct _zend_jit_registers_buf {
#if defined(IR_TARGET_X64)
uint64_t gpr[16]; /* general purpose integer register */
double fpr[16]; /* floating point registers */
#elif defined(IR_TARGET_X86)
uint32_t gpr[8]; /* general purpose integer register */
double fpr[8]; /* floating point registers */
#elif defined (IR_TARGET_AARCH64)
uint64_t gpr[32]; /* general purpose integer register */
double fpr[32]; /* floating point registers */
#else
# error "Unknown IR target"
#endif
} zend_jit_registers_buf;
/* Keep 32 exit points in a single code block */
#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
static uint32_t zend_jit_exit_point_by_addr(const void *addr);
int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs);
static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
zend_jit_addr ref_addr,
bool check_exception);
typedef struct _zend_jit_stub {
const char *name;
int (*stub)(zend_jit_ctx *jit);
uint32_t flags;
} zend_jit_stub;
JIT_STUBS(JIT_STUB_FORWARD)
static const zend_jit_stub zend_jit_stubs[] = {
JIT_STUBS(JIT_STUB)
};
#if defined(_WIN32) || defined(IR_TARGET_AARCH64)
/* We keep addresses in SHM to share them between sepaeate processes (on Windows) or to support veneers (on AArch64) */
static void** zend_jit_stub_handlers = NULL;
#else
static void* zend_jit_stub_handlers[sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])];
#endif
#if defined(IR_TARGET_AARCH64)
# ifdef __FreeBSD__
/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */
typedef struct TLSDescriptor {
void* thunk;
int index;
size_t offset;
} TLSDescriptor;
# endif
#define IR_HAS_VENEERS (1U<<31) /* IR_RESERVED_FLAG_1 */
static const void *zend_jit_get_veneer(ir_ctx *ctx, const void *addr)
{
int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
for (i = 0; i < count; i++) {
if (zend_jit_stub_handlers[i] == addr) {
return zend_jit_stub_handlers[count + i];
}
}
if (((zend_jit_ctx*)ctx)->trace
&& (void*)addr >= dasm_buf && (void*)addr < dasm_end) {
uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
if (exit_point != (uint32_t)-1) {
zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
ZEND_ASSERT(exit_point < t->exit_count);
return (const void*)((char*)ctx->deoptimization_exits_base + (exit_point * 4));
}
}
return NULL;
}
static bool zend_jit_set_veneer(ir_ctx *ctx, const void *addr, const void *veneer)
{
int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
uint32_t exit_point = zend_jit_exit_point_by_addr(addr);
if (exit_point != (uint32_t)-1) {
return 1;
}
for (i = 0; i < count; i++) {
if (zend_jit_stub_handlers[i] == addr) {
const void **ptr = (const void**)&zend_jit_stub_handlers[count + i];
*ptr = veneer;
ctx->flags2 |= IR_HAS_VENEERS;
#ifdef HAVE_CAPSTONE
int64_t offset;
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
const char *name = ir_disasm_find_symbol((uint64_t)(uintptr_t)addr, &offset);
if (name && !offset) {
if (strstr(name, "@veneer") == NULL) {
char *new_name;
zend_spprintf(&new_name, 0, "%s@veneer", name);
ir_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
efree(new_name);
} else {
ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
}
}
}
#endif
return 1;
}
}
return 0;
}
static void zend_jit_commit_veneers(void)
{
int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]);
for (i = 0; i < count; i++) {
if (zend_jit_stub_handlers[count + i]) {
zend_jit_stub_handlers[i] = zend_jit_stub_handlers[count + i];
zend_jit_stub_handlers[count + i] = NULL;
}
}
}
#endif
static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr)
{
#if defined(IR_TARGET_X86)
return 0; /* always use immediate value */
#elif defined(IR_TARGET_X64)
return addr > 0xffffffff; /* prefer loading long constant from memery */
#elif defined(IR_TARGET_AARCH64)
return addr > 0xffff;
#else
# error "Unknown IR target"
#endif
}
static const char* zend_reg_name(int8_t reg)
{
return ir_reg_name(reg, ir_reg_is_int(reg) ? IR_LONG : IR_DOUBLE);
}
/* IR helpers */
#ifdef ZTS
static ir_ref jit_TLS(zend_jit_ctx *jit)
{
ZEND_ASSERT(jit->ctx.control);
if (jit->tls) {
/* Emit "TLS" once for basic block */
ir_insn *insn;
ir_ref ref = jit->ctx.control;
while (1) {
if (ref == jit->tls) {
return jit->tls;
}
insn = &jit->ctx.ir_base[ref];
if (insn->op >= IR_START || insn->op == IR_CALL) {
break;
}
ref = insn->op1;
}
}
jit->tls = ir_TLS(
tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index,
tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset);
return jit->tls;
}
#endif
static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr)
{
ir_ref ref;
zval *zv;
if (addr == 0) {
return IR_NULL;
}
zv = zend_hash_index_lookup(&jit->addr_hash, addr);
if (Z_TYPE_P(zv) == IS_LONG) {
ref = Z_LVAL_P(zv);
ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR));
} else {
ref = ir_unique_const_addr(&jit->ctx, addr);
ZVAL_LONG(zv, ref);
}
return ref;
}
static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto)
{
ir_ref ref;
ir_insn *insn;
zval *zv;
ZEND_ASSERT(addr != 0);
zv = zend_hash_index_lookup(&jit->addr_hash, addr);
if (Z_TYPE_P(zv) == IS_LONG) {
ref = Z_LVAL_P(zv);
ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].proto == proto);
} else {
ref = ir_unique_const_addr(&jit->ctx, addr);
insn = &jit->ctx.ir_base[ref];
insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
insn->proto = proto;
ZVAL_LONG(zv, ref);
}
return ref;
}
static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags)
{
#if defined(IR_TARGET_X86)
/* TODO: dummy prototype (only flags matter) ??? */
ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
#else
ir_ref proto = 0;
#endif
return jit_CONST_FUNC_PROTO(jit, addr, proto);
}
static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset)
{
if (offset) {
addr = ir_ADD_A(addr, ir_CONST_ADDR(offset));
}
return addr;
}
static ir_ref jit_EG_exception(zend_jit_ctx *jit)
{
#ifdef ZTS
return jit_EG(exception);
#else
ir_ref ref = jit->eg_exception_addr;
if (UNEXPECTED(!ref)) {
ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception));
jit->eg_exception_addr = ref;
}
return ref;
#endif
}
static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id)
{
ir_ref ref = jit->stub_addr[id];
if (UNEXPECTED(!ref)) {
ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
jit->stub_addr[id] = ref;
}
return ref;
}
static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags)
{
ir_ref ref = jit->stub_addr[id];
ir_insn *insn;
if (UNEXPECTED(!ref)) {
ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]);
insn = &jit->ctx.ir_base[ref];
insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR);
#if defined(IR_TARGET_X86)
/* TODO: dummy prototype (only flags matter) ??? */
insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0;
#else
insn->proto = 0;
#endif
jit->stub_addr[id] = ref;
}
return ref;
}
static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr)
{
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr;
const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array;
uint32_t stack_size = op_array->last_var + op_array->T;
if (ptr == zend_jit_stub_handlers[jit_stub_exception_handler]
|| ptr == zend_jit_stub_handlers[jit_stub_exception_handler_undef]
|| ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op1_op2]
|| ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op2]
|| ptr == zend_jit_stub_handlers[jit_stub_interrupt_handler]
|| ptr == zend_jit_stub_handlers[jit_stub_leave_function_handler]
|| ptr == zend_jit_stub_handlers[jit_stub_negative_shift]
|| ptr == zend_jit_stub_handlers[jit_stub_mod_by_zero]
|| ptr == zend_jit_stub_handlers[jit_stub_invalid_this]
|| ptr == zend_jit_stub_handlers[jit_stub_undefined_function]
|| ptr == zend_jit_stub_handlers[jit_stub_throw_cannot_pass_by_ref]
|| ptr == zend_jit_stub_handlers[jit_stub_icall_throw]
|| ptr == zend_jit_stub_handlers[jit_stub_leave_throw]
|| ptr == zend_jit_stub_handlers[jit_stub_trace_halt]
|| ptr == zend_jit_stub_handlers[jit_stub_trace_escape]) {
/* This is a GUARD that trigger exit through a stub code (without deoptimization) */
return;
}
/* Check if we need snapshot entries for polymorphic method call */
zend_jit_trace_info *t = jit->trace;
uint32_t exit_point = 0, n = 0;
if (addr < 0) {
if (t->exit_count > 0
&& jit->ctx.ir_base[addr].val.u64 == (uintptr_t)zend_jit_trace_get_exit_addr(t->exit_count - 1)) {
exit_point = t->exit_count - 1;
if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) {
n = 2;
}
}
}
if (stack_size || n) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t snapshot_size, i;
snapshot_size = stack_size;
while (snapshot_size > 0) {
ir_ref ref = STACK_REF(stack, snapshot_size - 1);
if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, snapshot_size - 1) & (/*ZREG_LOAD|*/ZREG_STORE))) {
snapshot_size--;
} else {
break;
}
}
if (snapshot_size || n) {
ir_ref snapshot;
snapshot = ir_SNAPSHOT(snapshot_size + n);
for (i = 0; i < snapshot_size; i++) {
ir_ref ref = STACK_REF(stack, i);
if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, i) & (/*ZREG_LOAD|*/ZREG_STORE))) {
ref = IR_UNUSED;
}
ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref);
}
if (n) {
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func_ref);
ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this_ref);
}
}
}
}
}
static int32_t _add_trace_const(zend_jit_trace_info *t, int64_t val)
{
int32_t i;
for (i = 0; i < t->consts_count; i++) {
if (t->constants[i].i == val) {
return i;
}
}
ZEND_ASSERT(i < 0x7fffffff);
t->consts_count = i + 1;
t->constants = erealloc(t->constants, (i + 1) * sizeof(zend_jit_exit_const));
t->constants[i].i = val;
return i;
}
uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint32_t exit_point, ir_ref snapshot_ref)
{
uint32_t stack_size, stack_offset;
uint32_t new_exit_point = t->exit_count;
if (new_exit_point >= ZEND_JIT_TRACE_MAX_EXITS) {
ctx->status = -ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS;
return exit_point;
}
t->exit_count++;
memcpy(&t->exit_info[new_exit_point], &t->exit_info[exit_point], sizeof(zend_jit_trace_exit_info));
stack_size = t->exit_info[new_exit_point].stack_size;
if (stack_size != 0) {
stack_offset = t->stack_map_size;
t->stack_map_size += stack_size;
// TODO: reduce number of reallocations ???
t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack));
memcpy(t->stack_map + stack_offset, t->stack_map + t->exit_info[new_exit_point].stack_offset, stack_size * sizeof(zend_jit_trace_stack));
t->exit_info[new_exit_point].stack_offset = stack_offset;
}
t->exit_info[new_exit_point].flags &= ~ZEND_JIT_EXIT_FIXED;
return new_exit_point;
}
void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr)
{
zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace;
uint32_t exit_point, exit_flags;
ir_ref n = snapshot->inputs_count;
ir_ref i;
exit_point = zend_jit_exit_point_by_addr(addr);
ZEND_ASSERT(exit_point < t->exit_count);
exit_flags = t->exit_info[exit_point].flags;
if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) {
int8_t *reg_ops = ctx->regs[snapshot_ref];
ZEND_ASSERT(reg_ops[n - 1] != -1 && reg_ops[n] != -1);
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->exit_info[exit_point].poly_func_reg != reg_ops[n - 1]
|| t->exit_info[exit_point].poly_this_reg != reg_ops[n])) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->exit_info[exit_point].poly_func_reg = reg_ops[n - 1];
t->exit_info[exit_point].poly_this_reg = reg_ops[n];
n -= 2;
}
for (i = 2; i <= n; i++) {
ir_ref ref = ir_insn_op(snapshot, i);
if (ref) {
int8_t *reg_ops = ctx->regs[snapshot_ref];
int8_t reg = reg_ops[i];
ir_ref var = i - 2;
ZEND_ASSERT(var < t->exit_info[exit_point].stack_size);
if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags == ZREG_ZVAL_COPY) {
ZEND_ASSERT(reg != ZREG_NONE);
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
} else if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_CONST) {
ZEND_ASSERT(t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_LONG ||
t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_DOUBLE);
if (ref > 0) {
if (reg != ZREG_NONE) {
if (reg & IR_REG_SPILL_LOAD) {
ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL));
/* spill slot on a CPU stack */
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->stack_map[t->exit_info[exit_point].stack_offset + var].ref != ref
|| t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
|| !(t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ZREG_SPILL_SLOT))) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref;
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT;
} else if (reg & IR_REG_SPILL_SPECIAL) {
/* spill slot on a VM stack */
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
|| t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE;
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
} else {
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg);
}
} else {
if ((exit_flags & ZEND_JIT_EXIT_FIXED)
&& (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE
|| t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) {
exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref);
addr = (void*)zend_jit_trace_get_exit_addr(exit_point);
exit_flags &= ~ZEND_JIT_EXIT_FIXED;
}
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY;
}
} else if (!(exit_flags & ZEND_JIT_EXIT_FIXED)) {
int32_t idx = _add_trace_const(t, ctx->ir_base[ref].val.i64);
t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_CONST;
t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = idx;
}
}
}
}
t->exit_info[exit_point].flags |= ZEND_JIT_EXIT_FIXED;
return addr;
}
static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr)
{
jit_SNAPSHOT(jit, addr);
ir_IJMP(addr);
}
/* PHP JIT helpers */
static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend_op *opline)
{
#if ZEND_DEBUG
return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_emalloc),
ir_CONST_ADDR(size),
op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
ir_CONST_U32(opline ? opline->lineno : 0),
IR_NULL,
ir_CONST_U32(0));
#elif defined(HAVE_BUILTIN_CONSTANT_P)
if (size > 24 && size <= 32) {
return ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_emalloc_32));
} else {
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
}
#else
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size));
#endif
}
static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, const zend_op *opline)
{
#if ZEND_DEBUG
return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_efree),
ptr,
op_array && op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL,
ir_CONST_U32(opline ? opline->lineno : 0),
IR_NULL,
ir_CONST_U32(0));
#elif defined(HAVE_BUILTIN_CONSTANT_P)
if (size > 24 && size <= 32) {
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree_32), ptr);
} else {
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
}
#else
return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr);
#endif
}
static ir_ref jit_FP(zend_jit_ctx *jit)
{
ZEND_ASSERT(jit->ctx.control);
if (jit->fp == IR_UNUSED) {
/* Emit "RLOAD FP" once for basic block */
jit->fp = ir_RLOAD_A(ZREG_FP);
} else {
ir_insn *insn;
ir_ref ref = jit->ctx.control;
while (1) {
if (ref == jit->fp) {
break;
}
insn = &jit->ctx.ir_base[ref];
if (insn->op >= IR_START || insn->op == IR_CALL) {
jit->fp = ir_RLOAD_A(ZREG_FP);
break;
}
ref = insn->op1;
}
}
return jit->fp;
}
static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref)
{
ir_RSTORE(ZREG_FP, ref);
jit->fp = IR_UNUSED;
}
static ir_ref jit_IP(zend_jit_ctx *jit)
{
return ir_RLOAD_A(ZREG_IP);
}
static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref)
{
ir_RSTORE(ZREG_IP, ref);
}
static ir_ref jit_IP32(zend_jit_ctx *jit)
{
return ir_RLOAD_U32(ZREG_IP);
}
static void jit_LOAD_IP(zend_jit_ctx *jit, ir_ref ref)
{
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ref);
} else {
ir_STORE(jit_EX(opline), ref);
}
}
static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target)
{
jit_LOAD_IP(jit, ir_CONST_ADDR(target));
}
static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit)
{
jit->use_last_valid_opline = 0;
jit->track_last_valid_opline = 1;
}
static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit)
{
if (jit->track_last_valid_opline) {
jit->use_last_valid_opline = 1;
jit->track_last_valid_opline = 0;
}
}
static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit)
{
return jit->use_last_valid_opline;
}
static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline)
{
if (!jit->reuse_ip) {
jit->track_last_valid_opline = 1;
jit->last_valid_opline = opline;
}
}
static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit)
{
jit->track_last_valid_opline = 0;
jit->last_valid_opline = NULL;
}
static void zend_jit_start_reuse_ip(zend_jit_ctx *jit)
{
zend_jit_reset_last_valid_opline(jit);
jit->reuse_ip = 1;
}
static int zend_jit_reuse_ip(zend_jit_ctx *jit)
{
if (!jit->reuse_ip) {
zend_jit_start_reuse_ip(jit);
// RX = EX(call);
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
}
return 1;
}
static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit)
{
jit->reuse_ip = 0;
}
static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level)
{
ir_ref rx, call;
if (call_level == 1) {
// JIT: call = NULL;
call = IR_NULL;
} else {
// JIT: call = EX(call);
call = ir_LOAD_A(jit_EX(call));
}
rx = jit_IP(jit);
// JIT: call->prev_execute_data = call;
ir_STORE(jit_CALL(rx, prev_execute_data), call);
// JIT: EX(call) = call;
ir_STORE(jit_EX(call), rx);
jit->delayed_call_level = 0;
delayed_call_chain = 0;
return 1;
}
static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target)
{
ir_ref ref;
ir_ref addr = IR_UNUSED;
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (jit->last_valid_opline) {
zend_jit_use_last_valid_opline(jit);
if (jit->last_valid_opline != target) {
if (GCC_GLOBAL_REGS) {
ref = jit_IP(jit);
} else {
addr = jit_EX(opline);
ref = ir_LOAD_A(addr);
}
if (target > jit->last_valid_opline) {
ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline);
} else {
ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target));
}
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ref);
} else {
ir_STORE(addr, ref);
}
}
} else {
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ir_CONST_ADDR(target));
} else {
ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
}
}
jit->reuse_ip = 0;
zend_jit_set_last_valid_opline(jit, target);
return 1;
}
static int zend_jit_set_ip_ex(zend_jit_ctx *jit, const zend_op *target, bool set_ip_reg)
{
if (!GCC_GLOBAL_REGS && set_ip_reg && !jit->last_valid_opline) {
/* Optimization to avoid duplicate constant load */
ir_STORE(jit_EX(opline), ir_HARD_COPY_A(ir_CONST_ADDR(target)));
return 1;
}
return zend_jit_set_ip(jit, target);
}
static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target)
{
if (jit->last_valid_opline == target) {
zend_jit_use_last_valid_opline(jit);
if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
} else {
ir_STORE(jit_EX(opline), ir_CONST_ADDR(target));
if (!GCC_GLOBAL_REGS) {
zend_jit_reset_last_valid_opline(jit);
}
}
}
static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_ADD_OFFSET(reg, Z_OFFSET(addr));
} else if (Z_MODE(addr) == IS_REF_ZVAL) {
return Z_IR_REF(addr);
} else {
ZEND_ASSERT(Z_MODE(addr) == IS_CONST_ZVAL);
return ir_CONST_ADDR(Z_ZV(addr));
}
}
static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type)));
}
static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U8(Z_TYPE_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type)));
} else {
return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type_flags)));
}
static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U8(Z_TYPE_FLAGS_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type_flags)));
} else {
return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)));
}
static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U32(Z_TYPE_INFO_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
return ir_LOAD_U32(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)));
} else {
return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info)
{
ir_STORE(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)), type_info);
}
static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info)
{
if (Z_MODE(addr) == IS_MEM_ZVAL) {
ir_ref reg;
ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
if (Z_REG(addr) == ZREG_FP) {
reg = jit_FP(jit);
} else if (Z_REG(addr) == ZREG_RX) {
reg = jit_IP(jit);
} else {
ZEND_UNREACHABLE();
}
ir_STORE(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)), type_info);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info);
}
}
static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info)
{
if (type_info < IS_STRING
&& Z_MODE(addr) == IS_MEM_ZVAL
&& Z_REG(addr) == ZREG_FP
&& JIT_G(current_frame)
&& STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(addr))) == type_info) {
/* type is already set */
return;
}
jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info));
}
static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type)
{
return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type));
}
static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
{
ZEND_ASSERT(type != IS_UNDEF);
return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)));
}
static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type)
{
ir_ref ref = jit_Z_TYPE(jit, addr);
if (type != IS_UNDEF) {
ref = ir_NE(ref, ir_CONST_U8(type));
}
return ir_IF(ref);
}
static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
{
ir_ref ref = jit_Z_TYPE(jit, addr);
if (type != IS_UNDEF) {
ir_GUARD(ir_EQ(ref, ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
}
static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr)
{
ir_ref ref = jit_Z_TYPE(jit, addr);
if (type != IS_UNDEF) {
ref = ir_NE(ref, ir_CONST_U8(type));
}
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr)
{
return ir_IF(jit_Z_TYPE_FLAGS(jit, addr));
}
static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref)
{
return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE)));
}
static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_L(ref);
}
static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_D(ref);
}
static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val)
{
if (jit->ctx.ir_base[val].op == IR_RLOAD) {
/* Deoptimization */
return 0;
}
// if (jit->ctx.ir_base[val].op == IR_LOAD
// && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
// && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
// && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr == (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)) {
// /* LOAD from the same location (the LOAD is pinned) */
// // TODO: should be anti-dependent with the following stores ???
// return 0;
// }
if (jit->ssa->vars[var].var < jit->current_op_array->last_var) {
/* IS_CV */
if (jit->ctx.ir_base[val].op == IR_LOAD
&& jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD
&& jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD
&& jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP
&& IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2)
&& jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr != (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)
&& EX_VAR_TO_NUM(jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr) < jit->current_op_array->last_var) {
/* binding between different CVs may cause spill conflict */
return 1;
} else if (jit->ssa->vars[var].definition >= 0
&& jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var
&& jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0
&& jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].no_val
&& jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi
&& (jit->ssa->cfg.blocks[jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi->block].flags & ZEND_BB_LOOP_HEADER)) {
/* Avoid moving spill store out of loop */
return 1;
}
return 0;
}
return 1;
}
static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val)
{
int var;
ZEND_ASSERT(Z_MODE(addr) == IS_REG);
var = Z_SSA_VAR(addr);
if (var == jit->delay_var) {
ir_refs_add(jit->delay_refs, val);
return;
}
ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL);
/* Negative "var" has special meaning for IR */
if (val > 0 && !zend_jit_spilling_may_cause_conflict(jit, var, val)) {
val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val);
}
jit->ra[var].ref = val;
if (jit->ra[var].flags & ZREG_FORWARD) {
zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain;
zend_basic_block *bb;
int n, j, *p;
ir_ref *q;
jit->ra[var].flags &= ~ZREG_FORWARD;
while (phi != NULL) {
zend_ssa_phi *dst_phi = phi;
int src_var = var;
if (dst_phi->pi >= 0) {
jit->ra[src_var].ref = val;
src_var = dst_phi->ssa_var;
if (!(jit->ra[src_var].flags & ZREG_FORWARD)) {
phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
continue;
}
dst_phi = jit->ssa->vars[src_var].phi_use_chain;
ZEND_ASSERT(dst_phi != NULL && "reg forwarding");
ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding");
jit->ra[src_var].flags &= ~ZREG_FORWARD;
}
if (jit->ra[dst_phi->ssa_var].ref > 0) {
ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref];
if (phi_insn->op == IR_PHI) {
// ZEND_ASSERT(ir_operands_count(ctx, phi_insn) == n + 1);
bb = &jit->ssa->cfg.blocks[dst_phi->block];
n = bb->predecessors_count;
for (j = 0, p = &dst_phi->sources[0], q = phi_insn->ops + 2; j < n; j++, p++, q++) {
if (*p == src_var) {
*q = val;
}
}
}
}
phi = zend_ssa_next_use_phi(jit->ssa, var, phi);
}
}
}
static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr)
{
int var = Z_SSA_VAR(addr);
ZEND_ASSERT(Z_MODE(addr) == IS_REG);
ZEND_ASSERT(jit->ra && jit->ra[var].ref);
if (jit->ra[var].ref == IR_NULL) {
zend_jit_addr mem_addr;
ir_ref ref;
ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD);
mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var));
if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) {
ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
} else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) {
ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr));
} else {
ZEND_UNREACHABLE();
}
zend_jit_def_reg(jit, addr, ref);
return ref;
}
return jit->ra[Z_SSA_VAR(addr)].ref;
}
static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi)
{
int src_var = phi->sources[0];
int dst_var = phi->ssa_var;
ZEND_ASSERT(phi->pi >= 0);
ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
ZEND_ASSERT(jit->ra[src_var].ref);
if (jit->ra[src_var].ref == IR_NULL) {
/* Not defined yet */
if (jit->ssa->vars[dst_var].use_chain < 0
&& jit->ssa->vars[dst_var].phi_use_chain) {
zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain;
if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) {
/* This is a Pi forwarded to Phi */
jit->ra[src_var].flags |= ZREG_FORWARD;
return;
}
}
ZEND_ASSERT(0 && "Not defined Pi source");
}
/* Reuse register */
zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var),
zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)));
}
static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
{
int dst_var = phi->ssa_var;
zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block];
int n = bb->predecessors_count;
int i;
ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE;
ir_ref merge = jit->bb_start_ref[phi->block];
ir_ref ref;
ir_ref old_insns_count = jit->ctx.insns_count;
ir_ref same_src_ref = IR_UNUSED;
bool phi_inputs_are_the_same = 1;
ZEND_ASSERT(phi->pi < 0);
ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
ZEND_ASSERT(merge);
ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN);
ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count);
ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1);
ir_set_op(&jit->ctx, ref, 1, merge);
for (i = 0; i < n; i++) {
int src_var = phi->sources[i];
ZEND_ASSERT(jit->ra[src_var].ref);
if (jit->ra[src_var].ref == IR_NULL) {
jit->ra[src_var].flags |= ZREG_FORWARD;
phi_inputs_are_the_same = 0;
} else {
ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var));
if (i == 0) {
same_src_ref = src_ref;
} else if (same_src_ref != src_ref) {
phi_inputs_are_the_same = 0;
}
ir_set_op(&jit->ctx, ref, i + 2, src_ref);
}
}
if (phi_inputs_are_the_same) {
ref = same_src_ref;
jit->ctx.insns_count = old_insns_count;
}
zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
}
static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_LONG(Z_LVAL_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_REG) {
return zend_jit_use_reg(jit, addr);
} else {
return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
{
if (Z_MODE(addr) == IS_REG) {
zend_jit_def_reg(jit, addr, lval);
} else {
ir_STORE(jit_ZVAL_ADDR(jit, addr), lval);
}
}
#if SIZEOF_ZEND_LONG == 4
static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_U32((Z_ZV(addr))->value.ww.w2);
} else {
return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)));
}
}
static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval)
{
ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval);
}
#endif
static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_DOUBLE(Z_DVAL_P(Z_ZV(addr)));
} else if (Z_MODE(addr) == IS_REG) {
return zend_jit_use_reg(jit, addr);
} else {
return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval)
{
if (Z_MODE(addr) == IS_REG) {
zend_jit_def_reg(jit, addr, dval);
} else {
ir_STORE(jit_ZVAL_ADDR(jit, addr), dval);
}
}
static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_A(ref);
}
static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr)
{
if (Z_MODE(addr) == IS_CONST_ZVAL) {
return ir_CONST_ADDR(Z_PTR_P(Z_ZV(addr)));
} else {
return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr));
}
}
static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr)
{
ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr);
}
static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref)
{
return ir_LOAD_U32(ref);
}
static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount)
{
ir_STORE(ref, ir_CONST_U32(refcount));
}
static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref)
{
ir_STORE(ref, ir_ADD_U32(ir_LOAD_U32(ref), ir_CONST_U32(1)));
}
static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref counter = ir_LOAD_U32(ref);
ir_STORE(ref, ir_ADD_U32(counter, ir_CONST_U32(2)));
}
static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref counter = ir_LOAD_U32(ref);
counter = ir_SUB_U32(counter, ir_CONST_U32(1));
ir_STORE(ref, counter);
return counter;
}
static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref)
{
return ir_IF(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_refcounted, gc.u.type_info))),
ir_CONST_U32(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT))));
}
static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t dst_def_info, zval *zv, bool addref)
{
ir_ref ref = IR_UNUSED;
if (Z_TYPE_P(zv) > IS_TRUE) {
if (Z_TYPE_P(zv) == IS_DOUBLE) {
jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv)));
} else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv)));
} else if (Z_TYPE_P(zv) == IS_LONG) {
jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv)));
} else {
ref = ir_CONST_ADDR(Z_PTR_P(zv));
jit_set_Z_PTR(jit, dst, ref);
if (addref && Z_REFCOUNTED_P(zv)) {
jit_GC_ADDREF(jit, ref);
}
}
}
if (Z_MODE(dst) != IS_REG) {
if (dst_def_info == MAY_BE_DOUBLE) {
if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
jit_set_Z_TYPE_INFO(jit, dst, Z_TYPE_INFO_P(zv));
}
}
}
static ir_ref jit_if_TYPED_REF(zend_jit_ctx *jit, ir_ref ref)
{
return ir_IF(ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr))));
}
static void jit_ZVAL_COPY(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, bool addref)
{
ir_ref ref = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, jit_Z_LVAL(jit, src));
} else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, jit_Z_DVAL(jit, src));
} else {
#if SIZEOF_ZEND_LONG == 4
if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
jit_set_Z_W2(jit, dst, jit_Z_W2(jit, src));
}
#endif
ref = jit_Z_PTR(jit, src);
jit_set_Z_PTR(jit, dst, ref);
}
}
if (has_concrete_type(src_info & MAY_BE_ANY)
&& (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
&& !(src_info & MAY_BE_GUARD)) {
if (Z_MODE(dst) != IS_REG
&& (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
uint8_t type = concrete_type(src_info);
jit_set_Z_TYPE_INFO(jit, dst, type);
}
} else {
ir_ref type = jit_Z_TYPE_INFO(jit, src);
jit_set_Z_TYPE_INFO_ex(jit, dst, type);
if (addref) {
if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
ir_ref if_refcounted = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
ir_IF_TRUE(if_refcounted);
}
jit_GC_ADDREF(jit, ref);
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
}
}
}
}
}
static void jit_ZVAL_COPY_2(zend_jit_ctx *jit, zend_jit_addr dst2, zend_jit_addr dst, uint32_t dst_info, zend_jit_addr src, uint32_t src_info, int addref)
{
ir_ref ref = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
ref = jit_Z_LVAL(jit, src);
jit_set_Z_LVAL(jit, dst, ref);
jit_set_Z_LVAL(jit, dst2, ref);
} else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
ref = jit_Z_DVAL(jit, src);
jit_set_Z_DVAL(jit, dst, ref);
jit_set_Z_DVAL(jit, dst2, ref);
} else {
#if SIZEOF_ZEND_LONG == 4
if (src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD)) {
ref = jit_Z_W2(jit, src);
jit_set_Z_W2(jit, dst, ref);
jit_set_Z_W2(jit, dst2, ref);
}
#endif
ref = jit_Z_PTR(jit, src);
jit_set_Z_PTR(jit, dst, ref);
jit_set_Z_PTR(jit, dst2, ref);
}
}
if (has_concrete_type(src_info & MAY_BE_ANY)
&& (src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))
&& !(src_info & MAY_BE_GUARD)) {
uint8_t type = concrete_type(src_info);
ir_ref type_ref = ir_CONST_U32(type);
if (Z_MODE(dst) != IS_REG
&& (dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
jit_set_Z_TYPE_INFO_ex(jit, dst, type_ref);
}
if (Z_MODE(dst2) != IS_REG) {
jit_set_Z_TYPE_INFO_ex(jit, dst2, type_ref);
}
} else {
ir_ref type = jit_Z_TYPE_INFO(jit, src);
jit_set_Z_TYPE_INFO_ex(jit, dst, type);
jit_set_Z_TYPE_INFO_ex(jit, dst2, type);
if (addref) {
if (src_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
ir_ref if_refcounted = IR_UNUSED;
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(0xff00)));
ir_IF_TRUE(if_refcounted);
}
if (addref == 2) {
jit_GC_ADDREF2(jit, ref);
} else {
jit_GC_ADDREF(jit, ref);
}
if (src_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_MERGE_WITH_EMPTY_FALSE(if_refcounted);
}
}
}
}
}
static void jit_ZVAL_DTOR(zend_jit_ctx *jit, ir_ref ref, uint32_t op_info, const zend_op *opline)
{
if (!((op_info) & MAY_BE_GUARD)
&& has_concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
uint8_t type = concrete_type((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
if (type == IS_STRING && !ZEND_DEBUG) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(_efree), ref);
return;
} else if (type == IS_ARRAY) {
if ((op_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
if (opline && ((op_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_array_destroy), ref);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_array_free), ref);
}
return;
} else if (type == IS_OBJECT) {
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
return;
}
}
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(rc_dtor_func), ref);
}
static void jit_ZVAL_PTR_DTOR(zend_jit_ctx *jit,
zend_jit_addr addr,
uint32_t op_info,
bool gc,
const zend_op *opline)
{
ir_ref ref, ref2;
ir_ref if_refcounted = IR_UNUSED;
ir_ref if_not_zero = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
if (op_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = jit_if_REFCOUNTED(jit, addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(end_inputs);
ir_IF_TRUE(if_refcounted);
}
ref = jit_Z_PTR(jit, addr);
ref2 = jit_GC_DELREF(jit, ref);
if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
if_not_zero = ir_IF(ref2);
ir_IF_FALSE(if_not_zero);
}
// zval_dtor_func(r);
jit_ZVAL_DTOR(jit, ref, op_info, opline);
if (if_not_zero) {
ir_END_list(end_inputs);
ir_IF_TRUE(if_not_zero);
}
}
if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
ir_ref if_may_not_leak;
if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
ir_ref if_ref, if_collectable;
if_ref = jit_if_Z_TYPE(jit, addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
if_collectable = jit_if_COLLECTABLE_ref(jit, ref2);
ir_IF_FALSE(if_collectable);
ir_END_list(end_inputs);
ir_IF_TRUE(if_collectable);
ref2 = jit_Z_PTR_ref(jit, ref2);
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
}
if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_TRUE(if_may_not_leak);
ir_END_list(end_inputs);
ir_IF_FALSE(if_may_not_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
}
}
static void jit_FREE_OP(zend_jit_ctx *jit,
uint8_t op_type,
znode_op op,
uint32_t op_info,
const zend_op *opline)
{
if (op_type & (IS_VAR|IS_TMP_VAR)) {
jit_ZVAL_PTR_DTOR(jit,
ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var),
op_info, 0, opline);
}
}
static void jit_OBJ_RELEASE(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref end_inputs = IR_UNUSED;
ir_ref if_not_zero, if_may_not_leak;
// JIT: if (GC_DELREF(obj) == 0) {
if_not_zero = ir_IF(jit_GC_DELREF(jit, ref));
ir_IF_FALSE(if_not_zero);
// JIT: zend_objects_store_del(obj)
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_objects_store_del), ref);
ir_END_list(end_inputs);
ir_IF_TRUE(if_not_zero);
if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_TRUE(if_may_not_leak);
ir_END_list(end_inputs);
ir_IF_FALSE(if_may_not_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
static void zend_jit_check_timeout(zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr)
{
ir_ref ref = ir_LOAD_U8(jit_EG(vm_interrupt));
if (exit_addr) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else if (!opline || jit->last_valid_opline == opline) {
ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
} else {
ir_ref if_timeout = ir_IF(ref);
ir_IF_TRUE_cold(if_timeout);
jit_LOAD_IP_ADDR(jit, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler));
ir_IF_FALSE(if_timeout);
}
}
/* stubs */
static int zend_jit_exception_handler_stub(zend_jit_ctx *jit)
{
const void *handler;
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
handler = zend_get_opcode_handler_func(EG(exception_op));
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
handler = EG(exception_op)->handler;
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
ir_ref ref, if_negative;
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), jit_FP(jit));
if_negative = ir_IF(ir_LT(ref, ir_CONST_U32(0)));
ir_IF_TRUE(if_negative);
ir_MERGE_WITH_EMPTY_FALSE(if_negative);
ref = ir_PHI_2(IR_I32, ref, ir_CONST_I32(1));
ir_RETURN(ref);
}
}
return 1;
}
static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit)
{
ir_ref ref, result_type, if_result_used;
ref = jit_EG(opline_before_exception);
result_type = ir_LOAD_U8(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result_type)));
if_result_used = ir_IF(ir_AND_U8(result_type, ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_result_used);
ref = ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_UNDEF));
ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit)
{
ir_ref ref, if_dtor;
zend_jit_addr var_addr;
ref = ir_LOAD_A(jit_EG(opline_before_exception));
if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op1_type))),
ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_dtor);
ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op1.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ref = ir_ADD_A(jit_FP(jit), ref);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
return 1;
}
static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit)
{
ir_ref ref, if_dtor;
zend_jit_addr var_addr;
ref = ir_LOAD_A(jit_EG(opline_before_exception));
if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op2_type))),
ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_dtor);
ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ref = ir_ADD_A(jit_FP(jit), ref);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
ir_MERGE_WITH_EMPTY_FALSE(if_dtor);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
return 1;
}
static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit)
{
ir_ref if_timeout, if_exception;
if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
ir_STORE(jit_EG(vm_interrupt), ir_CONST_U8(0));
if_timeout = ir_IF(ir_EQ(ir_LOAD_U8(jit_EG(timed_out)), ir_CONST_U8(0)));
ir_IF_FALSE(if_timeout);
ir_CALL(IR_VOID, ir_CONST_FUNC(zend_timeout));
ir_MERGE_WITH_EMPTY_TRUE(if_timeout);
if (zend_interrupt_function) {
ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit));
if_exception = ir_IF(ir_LOAD_A(jit_EG(exception)));
ir_IF_TRUE(if_exception);
ir_CALL(IR_VOID, ir_CONST_FUNC(zend_jit_exception_in_interrupt_handler_helper));
ir_MERGE_WITH_EMPTY_FALSE(if_exception);
jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
}
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1));
}
return 1;
}
static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit)
{
ir_ref call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
ir_ref if_top = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP)));
ir_IF_FALSE(if_top);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
jit_STORE_IP(jit,
ir_LOAD_A(jit_EX(opline)));
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else if (GCC_GLOBAL_REGS) {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info);
} else {
ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info, jit_FP(jit));
}
ir_IF_TRUE(if_top);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else if (GCC_GLOBAL_REGS) {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info);
} else {
ir_TAILCALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info, jit_FP(jit));
}
return 1;
}
static int zend_jit_negative_shift_stub(zend_jit_ctx *jit)
{
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
ir_CONST_ADDR(zend_ce_arithmetic_error),
ir_CONST_ADDR("Bit shift by negative number"));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
return 1;
}
static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit)
{
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
ir_CONST_ADDR(zend_ce_division_by_zero_error),
ir_CONST_ADDR("Modulo by zero"));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2));
return 1;
}
static int zend_jit_invalid_this_stub(zend_jit_ctx *jit)
{
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
IR_NULL,
ir_CONST_ADDR("Using $this when not in object context"));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
return 1;
}
static int zend_jit_undefined_function_stub(zend_jit_ctx *jit)
{
// JIT: load EX(opline)
ir_ref ref = ir_LOAD_A(jit_FP(jit));
ir_ref arg3 = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.constant)));
if (sizeof(void*) == 8) {
arg3 = ir_LOAD_A(ir_ADD_A(ref, ir_SEXT_A(arg3)));
} else {
arg3 = ir_LOAD_A(arg3);
}
arg3 = ir_ADD_OFFSET(arg3, offsetof(zend_string, val));
ir_CALL_3(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
IR_NULL,
ir_CONST_ADDR("Call to undefined function %s()"),
arg3);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit)
{
ir_ref opline, ref, rx, if_eq, if_tmp;
// JIT: opline = EX(opline)
opline = ir_LOAD_A(jit_FP(jit));
// JIT: ZVAL_UNDEF(ZEND_CALL_VAR(RX, opline->result.var))
ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
rx = jit_IP(jit);
jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF));
// last EX(call) frame may be delayed
// JIT: if (EX(call) == RX)
ref = ir_LOAD_A(jit_EX(call));
if_eq = ir_IF(ir_EQ(rx, ref));
ir_IF_FALSE(if_eq);
// JIT: RX->prev_execute_data == EX(call)
ir_STORE(jit_CALL(rx, prev_execute_data), ref);
// JIT: EX(call) = RX
ir_STORE(jit_EX(call), rx);
ir_MERGE_WITH_EMPTY_TRUE(if_eq);
// JIT: IP = opline
jit_STORE_IP(jit, opline);
// JIT: zend_cannot_pass_by_reference(opline->op2.num)
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_cannot_pass_by_reference),
ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, op2.num))));
// JIT: if (IP->op1_type == IS_TMP_VAR)
ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type)));
if_tmp = ir_IF(ir_EQ(ref, ir_CONST_U8(IS_TMP_VAR)));
ir_IF_TRUE(if_tmp);
// JIT: zval_ptr_dtor(EX_VAR(IP->op1.var))
ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
ref = ir_ADD_A(jit_FP(jit), ref);
jit_ZVAL_PTR_DTOR(jit,
ZEND_ADDR_REF_ZVAL(ref),
MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL);
ir_MERGE_WITH_EMPTY_FALSE(if_tmp);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_icall_throw_stub(zend_jit_ctx *jit)
{
ir_ref ip, if_set;
// JIT: zend_rethrow_exception(zend_execute_data *execute_data)
// JIT: if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
ip = jit_IP(jit);
if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
ir_IF_FALSE(if_set);
// JIT: EG(opline_before_exception) = opline;
ir_STORE(jit_EG(opline_before_exception), ip);
ir_MERGE_WITH_EMPTY_TRUE(if_set);
// JIT: opline = EG(exception_op);
jit_STORE_IP(jit, jit_EG(exception_op));
if (GCC_GLOBAL_REGS) {
ir_STORE(jit_EX(opline), jit_IP(jit));
}
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
return 1;
}
static int zend_jit_leave_throw_stub(zend_jit_ctx *jit)
{
ir_ref ip, if_set;
// JIT: if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
ip = jit_IP(jit);
if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))),
ir_CONST_U8(ZEND_HANDLE_EXCEPTION)));
ir_IF_FALSE(if_set);
// JIT: EG(opline_before_exception) = opline;
ir_STORE(jit_EG(opline_before_exception), ip);
ir_MERGE_WITH_EMPTY_TRUE(if_set);
// JIT: opline = EG(exception_op);
jit_LOAD_IP(jit, jit_EG(exception_op));
if (GCC_GLOBAL_REGS) {
ir_STORE(jit_EX(opline), jit_IP(jit));
// JIT: HANDLE_EXCEPTION()
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
} else {
ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
}
return 1;
}
static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 0;
}
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_runtime_jit));
ir_IJMP(ir_LOAD_A(jit_IP(jit)));
return 1;
}
static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit)
{
ir_ref addr, func, run_time_cache, jit_extension;
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 0;
}
addr = ir_CONST_ADDR(&zend_jit_profile_counter),
ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
func = ir_LOAD_A(jit_EX(func));
run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
if (zend_jit_profile_counter_rid) {
addr = ir_ADD_OFFSET(run_time_cache, zend_jit_profile_counter_rid * sizeof(void*));
} else {
addr = run_time_cache;
}
ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1)));
addr = ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_extension, orig_handler));
ir_IJMP(ir_LOAD_A(addr));
return 1;
}
static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost)
{
ir_ref func, jit_extension, addr, ref, if_overflow;
func = ir_LOAD_A(jit_EX(func));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
addr = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, counter)));
ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
ir_STORE(addr, ref);
if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
ir_IF_TRUE_cold(if_overflow);
ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_hot_func),
jit_FP(jit),
jit_IP(jit));
ir_IJMP(ir_LOAD_A(jit_IP(jit)));
ir_IF_FALSE(if_overflow);
ref = ir_SUB_A(jit_IP(jit),
ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, opcodes))));
ref = ir_DIV_A(ref, ir_CONST_ADDR(sizeof(zend_op) / sizeof(void*)));
addr = ir_ADD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, orig_handlers)),
ref);
ir_IJMP(ir_LOAD_A(addr));
return 1;
}
static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 0;
}
return _zend_jit_hybrid_hot_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 0;
}
return _zend_jit_hybrid_hot_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset)
{
ir_ref addr;
if (GCC_GLOBAL_REGS) {
addr = ir_ADD_A(offset, jit_IP(jit));
} else {
addr = ir_ADD_A(offset, ir_LOAD_A(jit_EX(opline)));
}
return ir_LOAD_A(addr);
}
static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit)
{
ir_ref func, jit_extension, offset;
func = ir_LOAD_A(jit_EX(func));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
return _zend_jit_orig_opline_handler(jit, offset);
}
static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost)
{
ir_ref func, jit_extension, offset, addr, ref, if_overflow, ret, if_halt;
func = ir_LOAD_A(jit_EX(func));
jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid])));
offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset)));
addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter)));
ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost));
ir_STORE(addr, ref);
if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0)));
ir_IF_TRUE_cold(if_overflow);
ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT));
ret = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_trace_hot_root),
jit_FP(jit),
jit_IP(jit));
if_halt = ir_IF(ir_LT(ret, ir_CONST_I32(0)));
ir_IF_FALSE(if_halt);
ref = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(ref));
ref = ir_LOAD_A(jit_EX(opline));
jit_STORE_IP(jit, ref);
ir_IJMP(ir_LOAD_A(jit_IP(jit)));
ir_IF_FALSE(if_overflow);
ir_IJMP(_zend_jit_orig_opline_handler(jit, offset));
ir_IF_TRUE(if_halt);
ir_IJMP(ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
return 1;
}
static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 0;
}
return _zend_jit_hybrid_trace_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
return 0;
}
return _zend_jit_hybrid_trace_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
}
static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 0;
}
return _zend_jit_hybrid_trace_counter_stub(jit,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_trace_halt_stub(zend_jit_ctx *jit)
{
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_halt_op->handler));
} else if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, IR_NULL);
ir_RETURN(IR_VOID);
} else {
ir_RETURN(ir_CONST_I32(-1)); // ZEND_VM_RETURN
}
return 1;
}
static int zend_jit_trace_escape_stub(zend_jit_ctx *jit)
{
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
}
return 1;
}
static int zend_jit_trace_exit_stub(zend_jit_ctx *jit)
{
ir_ref ref, ret, if_zero, addr;
if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
ret = ir_EXITCALL(ir_CONST_FC_FUNC(zend_jit_trace_exit));
if_zero = ir_IF(ir_EQ(ret, ir_CONST_I32(0)));
ir_IF_TRUE(if_zero);
if (GCC_GLOBAL_REGS) {
ref = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(ref));
ref = ir_LOAD_A(jit_EX(opline));
jit_STORE_IP(jit, ref);
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
}
ir_IF_FALSE(if_zero);
ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
ref = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(ref));
if (GCC_GLOBAL_REGS) {
ref = ir_LOAD_A(jit_EX(opline));
jit_STORE_IP(jit, ref);
}
// check for interrupt (try to avoid this ???)
zend_jit_check_timeout(jit, NULL, NULL);
addr = zend_jit_orig_opline_handler(jit);
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, addr);
} else {
#if defined(IR_TARGET_X86)
addr = ir_CAST_FC_FUNC(addr);
#endif
ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
ir_RETURN(ir_CONST_I32(1)); // ZEND_VM_ENTER
}
return 1;
}
static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit)
{
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key));
} else {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit));
}
return 1;
}
static int zend_jit_undefined_key_stub(zend_jit_ctx *jit)
{
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key));
} else {
ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit));
}
return 1;
}
static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit)
{
ir_ref opline = ir_LOAD_A(jit_EX(opline));
ir_ref ref, if_result_used;
if_result_used = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(opline, offsetof(zend_op, result_type))),
ir_CONST_U8(IS_TMP_VAR|IS_VAR)));
ir_IF_TRUE(if_result_used);
ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF));
ir_MERGE_WITH_EMPTY_FALSE(if_result_used);
ir_CALL_2(IR_VOID,
ir_CONST_FUNC_PROTO(zend_throw_error,
ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)),
IR_NULL,
ir_CONST_ADDR("Cannot add element to the array as the next element is already occupied"));
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_const_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_CONST, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_TMP_VAR, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_var_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_VAR, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static int zend_jit_new_array_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
ir_ref ref = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_zend_new_array_0));
jit_set_Z_PTR(jit, var_addr, ref);
jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX);
ir_RETURN(ref);
return 1;
}
static int zend_jit_assign_cv_stub(zend_jit_ctx *jit)
{
ir_ref var = ir_PARAM(IR_ADDR, "var", 1);
ir_ref val = ir_PARAM(IR_ADDR, "val", 2);
zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
if (!zend_jit_assign_to_variable(
jit, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0, 0)) {
return 0;
}
ir_RETURN(IR_VOID);
return 1;
}
static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags)
{
#if defined (__CET__) && (__CET__ & 1) != 0
flags |= IR_GEN_ENDBR;
#endif
flags |= IR_OPT_FOLDING | IR_OPT_CFG | IR_OPT_CODEGEN;
ir_init(&jit->ctx, flags, 256, 1024);
jit->ctx.ret_type = -1;
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
jit->ctx.mflags |= default_mflags;
if (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) {
jit->ctx.mflags |= IR_X86_AVX;
}
#elif defined(IR_TARGET_AARCH64)
jit->ctx.get_veneer = zend_jit_get_veneer;
jit->ctx.set_veneer = zend_jit_set_veneer;
#endif
jit->ctx.fixed_regset = (1<<ZREG_FP) | (1<<ZREG_IP);
if (!(flags & IR_FUNCTION)) {
jit->ctx.flags |= IR_NO_STACK_COMBINE;
if (zend_jit_vm_kind == ZEND_VM_KIND_CALL) {
jit->ctx.flags |= IR_FUNCTION;
/* Stack must be 16 byte aligned */
/* TODO: select stack size ??? */
#if defined(IR_TARGET_AARCH64)
jit->ctx.flags |= IR_USE_FRAME_POINTER;
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 bytes) */
#elif defined(_WIN64)
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 bytes) */
#elif defined(IR_TARGET_X86_64)
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 bytes) */
#else /* IR_TARGET_x86 */
jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 bytes) */
#endif
if (GCC_GLOBAL_REGS) {
jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<<ZREG_FP) | (1<<ZREG_IP));
} else {
jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED;
//#ifdef _WIN64
// jit->ctx.fixed_save_regset &= 0xffff; // TODO: don't save FP registers ???
//#endif
}
jit->ctx.fixed_call_stack_size = 16;
} else {
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE;
if (jit->ctx.fixed_stack_red_zone > 16) {
jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16;
jit->ctx.fixed_call_stack_size = 16;
}
jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES;
#else
jit->ctx.fixed_stack_red_zone = 0;
jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */
jit->ctx.fixed_call_stack_size = 16;
#endif
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
jit->ctx.fixed_regset |= (1<<IR_REG_FP); /* prevent %rbp (%r5) usage */
#endif
}
}
jit->ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT;
jit->op_array = NULL;
jit->current_op_array = NULL;
jit->ssa = NULL;
jit->name = NULL;
jit->last_valid_opline = NULL;
jit->use_last_valid_opline = 0;
jit->track_last_valid_opline = 0;
jit->reuse_ip = 0;
jit->delayed_call_level = 0;
delayed_call_chain = 0;
jit->b = -1;
#ifdef ZTS
jit->tls = IR_UNUSED;
#endif
jit->fp = IR_UNUSED;
jit->trace_loop_ref = IR_UNUSED;
jit->return_inputs = IR_UNUSED;
jit->bb_start_ref = NULL;
jit->bb_predecessors = NULL;
jit->bb_edges = NULL;
jit->trace = NULL;
jit->ra = NULL;
jit->delay_var = -1;
jit->delay_refs = NULL;
jit->eg_exception_addr = 0;
zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0);
memset(jit->stub_addr, 0, sizeof(jit->stub_addr));
ir_START();
}
static int zend_jit_free_ctx(zend_jit_ctx *jit)
{
if (jit->name) {
zend_string_release(jit->name);
}
zend_hash_destroy(&jit->addr_hash);
ir_free(&jit->ctx);
return 1;
}
static void *zend_jit_ir_compile(ir_ctx *ctx, size_t *size, const char *name)
{
void *entry;
ir_code_buffer code_buffer;
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_SRC) {
if (name) fprintf(stderr, "%s: ; after folding\n", name);
ir_save(ctx, 0, stderr);
}
#if ZEND_DEBUG
ir_check(ctx);
#endif
ir_build_def_use_lists(ctx);
#if ZEND_DEBUG
ir_check(ctx);
#endif
#if 1
ir_sccp(ctx);
#endif
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCCP) {
if (name) fprintf(stderr, "%s: ; after SCCP\n", name);
ir_save(ctx, 0, stderr);
}
ir_build_cfg(ctx);
ir_build_dominators_tree(ctx);
ir_find_loops(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_CFG) {
if (name) fprintf(stderr, "%s: ; after CFG\n", name);
ir_save(ctx, IR_SAVE_CFG, stderr);
}
ir_gcm(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_GCM) {
if (name) fprintf(stderr, "%s: ; after GCM\n", name);
ir_save(ctx, IR_SAVE_CFG|IR_SAVE_CFG_MAP, stderr);
}
ir_schedule(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE) {
if (name) fprintf(stderr, "%s: ; after schedule\n", name);
ir_save(ctx, IR_SAVE_CFG, stderr);
}
ir_match(ctx);
#if !defined(IR_TARGET_AARCH64)
ctx->flags &= ~IR_USE_FRAME_POINTER; /* don't use FRAME_POINTER even with ALLOCA, TODO: cleanup this ??? */
#endif
ir_assign_virtual_registers(ctx);
ir_compute_live_ranges(ctx);
ir_coalesce(ctx);
ir_reg_alloc(ctx);
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_REGS) {
if (name) fprintf(stderr, "%s: ; after register allocation\n", name);
ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
ir_dump_live_ranges(ctx, stderr);
}
ir_schedule_blocks(ctx);
if (JIT_G(debug) & (ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_CODEGEN) {
if (name) fprintf(stderr, "%s: ; codegen\n", name);
ir_dump_codegen(ctx, stderr);
} else {
if (name) fprintf(stderr, "%s: ; final\n", name);
ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr);
}
}
#if ZEND_DEBUG
ir_check(ctx);
#endif
code_buffer.start = dasm_buf;
code_buffer.end = dasm_end;
code_buffer.pos = *dasm_ptr;
ctx->code_buffer = &code_buffer;
entry = ir_emit_code(ctx, size);
*dasm_ptr = code_buffer.pos;
#if defined(IR_TARGET_AARCH64)
if (ctx->flags2 & IR_HAS_VENEERS) {
zend_jit_commit_veneers();
}
#endif
return entry;
}
static void zend_jit_setup_stubs(void)
{
zend_jit_ctx jit;
void *entry;
size_t size;
uint32_t i;
for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) {
zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags);
if (!zend_jit_stubs[i].stub(&jit)) {
zend_jit_free_ctx(&jit);
zend_jit_stub_handlers[i] = NULL;
continue;
}
entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name);
if (!entry) {
zend_jit_free_ctx(&jit);
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile stub");
}
zend_jit_stub_handlers[i] = entry;
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
ir_disasm_add_symbol(zend_jit_stubs[i].name, (uintptr_t)entry, size);
}
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) {
ir_disasm(zend_jit_stubs[i].name,
entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr);
}
#endif
#ifndef _WIN32
if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
// ir_mem_unprotect(entry, size);
ir_gdb_register(zend_jit_stubs[i].name, entry, size, 0, 0);
// ir_mem_protect(entry, size);
}
if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
ir_perf_map_register(zend_jit_stubs[i].name, entry, size);
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_register(zend_jit_stubs[i].name, entry, size);
}
}
#endif
}
zend_jit_free_ctx(&jit);
}
}
#define REGISTER_HELPER(n) \
ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)n, sizeof(void*));
#define REGISTER_DATA(n) \
ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)&n, sizeof(n));
static void zend_jit_setup_disasm(void)
{
#ifdef HAVE_CAPSTONE
ir_disasm_init();
if (zend_vm_kind() == ZEND_VM_KIND_HYBRID) {
zend_op opline;
memset(&opline, 0, sizeof(opline));
opline.opcode = ZEND_DO_UCALL;
opline.result_type = IS_UNUSED;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_UCALL;
opline.result_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL_BY_NAME;
opline.result_type = IS_UNUSED;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL_BY_NAME;
opline.result_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL;
opline.result_type = IS_UNUSED;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_DO_FCALL;
opline.result_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_CONST;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_CONST_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_TMP_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_TMP_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_VAR;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_VAR_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
opline.opcode = ZEND_RETURN;
opline.op1_type = IS_CV;
zend_vm_set_opcode_handler(&opline);
ir_disasm_add_symbol("ZEND_RETURN_SPEC_CV_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*));
ir_disasm_add_symbol("ZEND_HYBRID_HALT_LABEL", (uint64_t)(uintptr_t)zend_jit_halt_op->handler, sizeof(void*));
}
REGISTER_DATA(zend_jit_profile_counter);
REGISTER_HELPER(zend_runtime_jit);
REGISTER_HELPER(zend_jit_hot_func);
REGISTER_HELPER(zend_jit_trace_hot_root);
REGISTER_HELPER(zend_jit_trace_exit);
REGISTER_HELPER(zend_jit_array_free);
REGISTER_HELPER(zend_jit_undefined_op_helper);
REGISTER_HELPER(zend_jit_pre_inc_typed_ref);
REGISTER_HELPER(zend_jit_pre_dec_typed_ref);
REGISTER_HELPER(zend_jit_post_inc_typed_ref);
REGISTER_HELPER(zend_jit_post_dec_typed_ref);
REGISTER_HELPER(zend_jit_pre_inc);
REGISTER_HELPER(zend_jit_pre_dec);
REGISTER_HELPER(zend_jit_add_arrays_helper);
REGISTER_HELPER(zend_jit_fast_assign_concat_helper);
REGISTER_HELPER(zend_jit_fast_concat_helper);
REGISTER_HELPER(zend_jit_fast_concat_tmp_helper);
REGISTER_HELPER(zend_jit_assign_op_to_typed_ref_tmp);
REGISTER_HELPER(zend_jit_assign_op_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_const_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_var_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref);
REGISTER_HELPER(zend_jit_assign_const_to_typed_ref2);
REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref2);
REGISTER_HELPER(zend_jit_assign_var_to_typed_ref2);
REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref2);
REGISTER_HELPER(zend_jit_check_constant);
REGISTER_HELPER(zend_jit_get_constant);
REGISTER_HELPER(zend_jit_int_extend_stack_helper);
REGISTER_HELPER(zend_jit_extend_stack_helper);
REGISTER_HELPER(zend_jit_init_func_run_time_cache_helper);
REGISTER_HELPER(zend_jit_find_func_helper);
REGISTER_HELPER(zend_jit_find_ns_func_helper);
REGISTER_HELPER(zend_jit_jmp_frameless_helper);
REGISTER_HELPER(zend_jit_unref_helper);
REGISTER_HELPER(zend_jit_invalid_method_call);
REGISTER_HELPER(zend_jit_invalid_method_call_tmp);
REGISTER_HELPER(zend_jit_find_method_helper);
REGISTER_HELPER(zend_jit_find_method_tmp_helper);
REGISTER_HELPER(zend_jit_push_static_method_call_frame);
REGISTER_HELPER(zend_jit_push_static_method_call_frame_tmp);
REGISTER_HELPER(zend_jit_find_class_helper);
REGISTER_HELPER(zend_jit_find_static_method_helper);
REGISTER_HELPER(zend_jit_push_this_method_call_frame);
REGISTER_HELPER(zend_jit_free_trampoline_helper);
REGISTER_HELPER(zend_jit_verify_return_slow);
REGISTER_HELPER(zend_jit_deprecated_helper);
REGISTER_HELPER(zend_jit_undefined_long_key);
REGISTER_HELPER(zend_jit_undefined_long_key_ex);
REGISTER_HELPER(zend_jit_undefined_string_key);
REGISTER_HELPER(zend_jit_copy_extra_args_helper);
REGISTER_HELPER(zend_jit_vm_stack_free_args_helper);
REGISTER_HELPER(zend_free_extra_named_params);
REGISTER_HELPER(zend_jit_free_call_frame);
REGISTER_HELPER(zend_jit_exception_in_interrupt_handler_helper);
REGISTER_HELPER(zend_jit_verify_arg_slow);
REGISTER_HELPER(zend_missing_arg_error);
REGISTER_HELPER(zend_jit_only_vars_by_reference);
REGISTER_HELPER(zend_jit_leave_func_helper);
REGISTER_HELPER(zend_jit_leave_nested_func_helper);
REGISTER_HELPER(zend_jit_leave_top_func_helper);
REGISTER_HELPER(zend_jit_fetch_global_helper);
REGISTER_HELPER(zend_jit_hash_index_lookup_rw_no_packed);
REGISTER_HELPER(zend_jit_hash_index_lookup_rw);
REGISTER_HELPER(zend_jit_hash_lookup_rw);
REGISTER_HELPER(zend_jit_symtable_find);
REGISTER_HELPER(zend_jit_symtable_lookup_w);
REGISTER_HELPER(zend_jit_symtable_lookup_rw);
REGISTER_HELPER(zend_jit_fetch_dim_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_is_helper);
REGISTER_HELPER(zend_jit_fetch_dim_isset_helper);
REGISTER_HELPER(zend_jit_fetch_dim_rw_helper);
REGISTER_HELPER(zend_jit_fetch_dim_w_helper);
REGISTER_HELPER(zend_jit_fetch_dim_str_offset_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper);
REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper);
REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper);
REGISTER_HELPER(zend_jit_invalid_array_access);
REGISTER_HELPER(zend_jit_zval_array_dup);
REGISTER_HELPER(zend_jit_prepare_assign_dim_ref);
REGISTER_HELPER(zend_jit_fetch_dim_obj_w_helper);
REGISTER_HELPER(zend_jit_fetch_dim_obj_rw_helper);
REGISTER_HELPER(zend_jit_isset_dim_helper);
REGISTER_HELPER(zend_jit_assign_dim_helper);
REGISTER_HELPER(zend_jit_assign_dim_op_helper);
REGISTER_HELPER(zend_jit_fetch_obj_w_slow);
REGISTER_HELPER(zend_jit_fetch_obj_r_slow);
REGISTER_HELPER(zend_jit_fetch_obj_r_slow_ex);
REGISTER_HELPER(zend_jit_fetch_obj_is_slow);
REGISTER_HELPER(zend_jit_fetch_obj_is_slow_ex);
REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic);
REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic_ex);
REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic);
REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic_ex);
REGISTER_HELPER(zend_jit_check_array_promotion);
REGISTER_HELPER(zend_jit_create_typed_ref);
REGISTER_HELPER(zend_jit_invalid_property_write);
REGISTER_HELPER(zend_jit_invalid_property_read);
REGISTER_HELPER(zend_jit_extract_helper);
REGISTER_HELPER(zend_jit_invalid_property_assign);
REGISTER_HELPER(zend_jit_assign_to_typed_prop);
REGISTER_HELPER(zend_jit_assign_obj_helper);
REGISTER_HELPER(zend_jit_invalid_property_assign_op);
REGISTER_HELPER(zend_jit_assign_op_to_typed_prop);
REGISTER_HELPER(zend_jit_assign_obj_op_helper);
REGISTER_HELPER(zend_jit_invalid_property_incdec);
REGISTER_HELPER(zend_jit_inc_typed_prop);
REGISTER_HELPER(zend_jit_dec_typed_prop);
REGISTER_HELPER(zend_jit_pre_inc_typed_prop);
REGISTER_HELPER(zend_jit_post_inc_typed_prop);
REGISTER_HELPER(zend_jit_pre_dec_typed_prop);
REGISTER_HELPER(zend_jit_post_dec_typed_prop);
REGISTER_HELPER(zend_jit_pre_inc_obj_helper);
REGISTER_HELPER(zend_jit_post_inc_obj_helper);
REGISTER_HELPER(zend_jit_pre_dec_obj_helper);
REGISTER_HELPER(zend_jit_post_dec_obj_helper);
REGISTER_HELPER(zend_jit_uninit_static_prop);
REGISTER_HELPER(zend_jit_rope_end);
REGISTER_HELPER(zend_fcall_interrupt);
#ifndef ZTS
REGISTER_DATA(EG(current_execute_data));
REGISTER_DATA(EG(exception));
REGISTER_DATA(EG(opline_before_exception));
REGISTER_DATA(EG(vm_interrupt));
REGISTER_DATA(EG(timed_out));
REGISTER_DATA(EG(uninitialized_zval));
REGISTER_DATA(EG(zend_constants));
REGISTER_DATA(EG(jit_trace_num));
REGISTER_DATA(EG(vm_stack_top));
REGISTER_DATA(EG(vm_stack_end));
REGISTER_DATA(EG(exception_op));
REGISTER_DATA(EG(symbol_table));
REGISTER_DATA(CG(map_ptr_base));
#endif
#endif
}
static void zend_jit_calc_trace_prologue_size(void)
{
zend_jit_ctx jit_ctx;
zend_jit_ctx *jit = &jit_ctx;
void *entry;
size_t size;
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
if (!GCC_GLOBAL_REGS) {
ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
jit_STORE_FP(jit, ref);
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
ir_UNREACHABLE();
entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue");
zend_jit_free_ctx(jit);
if (!entry) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile prologue");
}
zend_jit_trace_prologue_size = size;
}
#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
static uintptr_t zend_jit_hybrid_vm_sp_adj = 0;
typedef struct _Unwind_Context _Unwind_Context;
typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
typedef struct _zend_jit_unwind_arg {
int cnt;
uintptr_t cfa[3];
} zend_jit_unwind_arg;
static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
{
zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
arg->cnt++;
if (arg->cnt == 3) {
return 5; // _URC_END_OF_STACK
}
return 0; // _URC_NO_REASON;
}
static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
{
zend_jit_unwind_arg arg;
memset(&arg, 0, sizeof(arg));
_Unwind_Backtrace(zend_jit_unwind_cb, &arg);
if (arg.cnt == 3) {
zend_jit_hybrid_vm_sp_adj = arg.cfa[2] - arg.cfa[1];
}
}
extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);
static zend_never_inline void zend_jit_set_sp_adj_vm(void)
{
void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);
orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
execute_ex(NULL); // set sp_adj[SP_ADJ_VM]
zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
}
#endif
static void zend_jit_setup(void)
{
#if defined(IR_TARGET_X86)
if (!zend_cpu_supports_sse2()) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: CPU doesn't support SSE2");
}
#endif
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
allowed_opt_flags = 0;
if (zend_cpu_supports_avx()) {
allowed_opt_flags |= ZEND_JIT_CPU_AVX;
}
# if defined(PHP_HAVE_BUILTIN_CPU_SUPPORTS) && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000)
if (zend_cpu_supports_cldemote()) {
default_mflags |= IR_X86_CLDEMOTE;
}
# endif
#endif
#ifdef ZTS
#if defined(IR_TARGET_AARCH64)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
# ifdef __FreeBSD__
if (tsrm_ls_cache_tcb_offset == 0) {
TLSDescriptor **where;
__asm__(
"adrp %0, :tlsdesc:_tsrm_ls_cache\n"
"add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n"
: "=r" (where));
/* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst
* section "Relocations for thread-local storage".
* The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds
* a platform-specific offset or pointer. */
TLSDescriptor *tlsdesc = where[1];
tsrm_tls_offset = tlsdesc->offset;
/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */
tsrm_tls_index = (tlsdesc->index + 1) * 8;
}
# else
ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0);
# endif
# elif defined(_WIN64)
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
}
} while(0);
# elif defined(ZEND_WIN32)
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: offset >= size");
}
} while(0);
# elif defined(__APPLE__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
size_t *ti;
__asm__(
"leaq __tsrm_ls_cache(%%rip),%0"
: "=r" (ti));
tsrm_tls_offset = ti[2];
tsrm_tls_index = ti[1] * 8;
}
# elif defined(__GNUC__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && \
!defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0"
: "=r" (ret));
tsrm_ls_cache_tcb_offset = ret;
#elif defined(__MUSL__)
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=a" (ti));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 8;
#elif defined(__FreeBSD__)
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=a" (ti));
tsrm_tls_offset = ti[1];
/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */
tsrm_tls_index = (ti[0] + 1) * 8;
#else
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=a" (ti));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 16;
#endif
}
# elif defined(__GNUC__) && defined(__i386__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
: "=a" (ret));
tsrm_ls_cache_tcb_offset = ret;
#else
size_t *ti, _ebx, _ecx, _edx;
__asm__(
"call 1f\n"
".subsection 1\n"
"1:\tmovl (%%esp), %%ebx\n\t"
"ret\n"
".previous\n\t"
"addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
"call ___tls_get_addr@plt\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
: "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 8;
#endif
}
# endif
#endif
#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
zend_jit_set_sp_adj_vm(); // set zend_jit_hybrid_vm_sp_adj
}
#endif
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
zend_jit_setup_disasm();
}
#ifndef _WIN32
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_open();
}
#endif
zend_long debug = JIT_G(debug);
if (!(debug & ZEND_JIT_DEBUG_ASM_STUBS)) {
JIT_G(debug) &= ~(ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_FINAL|
ZEND_JIT_DEBUG_IR_CODEGEN|
ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_CFG|ZEND_JIT_DEBUG_IR_AFTER_GCM|
ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS);
}
zend_jit_calc_trace_prologue_size();
zend_jit_setup_stubs();
JIT_G(debug) = debug;
}
static void zend_jit_shutdown_ir(void)
{
#ifndef _WIN32
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_close();
}
if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
ir_gdb_unregister_all();
}
#endif
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) {
ir_disasm_free();
}
#endif
}
/* PHP control flow reconstruction helpers */
static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block)
{
ir_ref ref = ir_IF(condition);
/* op3 is used as a temporary storage for PHP BB number to reconstruct PHP control flow.
*
* It's used in jit_IF_TRUE_FALSE_ex() to select IF_TRUE or IF_FALSE instructions
* to start target block
*/
ir_set_op(&jit->ctx, ref, 3, true_block);
return ref;
}
static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block)
{
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
ZEND_ASSERT(if_ref);
ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF);
ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3);
if (jit->ctx.ir_base[if_ref].op3 == true_block) {
ir_IF_TRUE(if_ref);
} else {
ir_IF_FALSE(if_ref);
}
}
static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref)
{
int i, *p;
zend_basic_block *bb;
ir_ref *r, header;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
bb = &jit->ssa->cfg.blocks[b];
p = &jit->ssa->cfg.predecessors[bb->predecessor_offset];
r = &jit->bb_edges[jit->bb_predecessors[b]];
for (i = 0; i < bb->predecessors_count; i++, p++, r++) {
if (*p == pred) {
ZEND_ASSERT(*r == IR_UNUSED || *r == ref);
header = jit->bb_start_ref[b];
if (header) {
/* this is back edge */
ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN);
if (jit->ctx.ir_base[ref].op == IR_END) {
jit->ctx.ir_base[ref].op = IR_LOOP_END;
} else if (jit->ctx.ir_base[ref].op == IR_IF) {
jit_IF_TRUE_FALSE_ex(jit, ref, b);
ref = ir_LOOP_END();
} else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) {
ir_BEGIN(ref);
ref = ir_LOOP_END();
} else {
ZEND_UNREACHABLE();
}
ir_MERGE_SET_OP(header, i + 1, ref);
}
*r = ref;
return;
}
}
ZEND_UNREACHABLE();
}
static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit,
uint32_t true_label,
uint32_t false_label,
ir_ref true_inputs,
ir_ref false_inputs)
{
ir_ref true_path = IR_UNUSED, false_path = IR_UNUSED;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
if (true_inputs) {
ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END);
if (!jit->ctx.ir_base[true_inputs].op2) {
true_path = true_inputs;
} else {
ir_MERGE_list(true_inputs);
true_path = ir_END();
}
}
if (false_inputs) {
ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END);
if (!jit->ctx.ir_base[false_inputs].op2) {
false_path = false_inputs;
} else {
ir_MERGE_list(false_inputs);
false_path = ir_END();
}
}
if (true_label == false_label && true_path && false_path) {
ir_MERGE_2(true_path, false_path);
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
} else if (!true_path && !false_path) {
/* dead code */
true_path = ir_END();
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
} else {
if (true_path) {
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path);
}
if (false_path) {
_zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path);
}
}
jit->b = -1;
}
static void _zend_jit_fix_merges(zend_jit_ctx *jit)
{
int i, count;
ir_ref j, k, n, *p, *q, *r;
ir_ref ref;
ir_insn *insn, *phi;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
count = jit->ssa->cfg.blocks_count;
for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) {
ref = *p;
if (ref) {
insn = &jit->ctx.ir_base[ref];
if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) {
n = insn->inputs_count;
/* Remove IS_UNUSED inputs */
for (j = k = 0, q = r = insn->ops + 1; j < n; j++, q++) {
if (*q) {
if (q != r) {
*r = *q;
phi = insn + 1 + (n >> 2);
while (phi->op == IR_PI) {
phi++;
}
while (phi->op == IR_PHI) {
ir_insn_set_op(phi, k + 2, ir_insn_op(phi, j + 2));
phi += 1 + ((n + 1) >> 2);
}
}
k++;
r++;
}
}
if (k != n) {
ir_ref n2, k2;
if (k <= 1) {
insn->op = IR_BEGIN;
insn->inputs_count = 0;
} else {
insn->inputs_count = k;
}
n2 = 1 + (n >> 2);
k2 = 1 + (k >> 2);
while (k2 != n2) {
(insn+k2)->optx = IR_NOP;
k2++;
}
phi = insn + 1 + (n >> 2);
while (phi->op == IR_PI) {
phi++;
}
while (phi->op == IR_PHI) {
if (k <= 1) {
phi->op = IR_COPY;
phi->op1 = phi->op2;
phi->op2 = 1;
phi->inputs_count = 0;
} else {
phi->inputs_count = k + 1;
}
n2 = 1 + ((n + 1) >> 2);
k2 = 1 + ((k + 1) >> 2);
while (k2 != n2) {
(phi+k2)->optx = IR_NOP;
k2++;
}
phi += 1 + ((n + 1) >> 2);
}
}
}
}
}
}
static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref)
{
zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b];
const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
if (opline->opcode == ZEND_SWITCH_LONG
|| opline->opcode == ZEND_SWITCH_STRING
|| opline->opcode == ZEND_MATCH) {
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes];
zval *zv;
ir_ref list = IR_UNUSED, idx;
bool first = 1;
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
int b = jit->ssa->cfg.map[target - jit->op_array->opcodes];
if (b == case_b) {
if (!first) {
ir_END_list(list);
}
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(switch_ref, idx);
first = 0;
}
} ZEND_HASH_FOREACH_END();
if (default_b == case_b) {
if (!first) {
ir_END_list(list);
}
if (jit->ctx.ir_base[switch_ref].op3) {
/* op3 may contain a list of additional "default" path inputs for MATCH */
ir_ref ref = jit->ctx.ir_base[switch_ref].op3;
jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF;
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
ir_ref end = ref;
while (jit->ctx.ir_base[end].op2) {
ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END);
end = jit->ctx.ir_base[end].op2;
}
jit->ctx.ir_base[end].op2 = list;
list = ref;
}
ir_CASE_DEFAULT(switch_ref);
}
if (list) {
ir_END_list(list);
ir_MERGE_list(list);
}
} else {
ZEND_UNREACHABLE();
}
}
static int zend_jit_bb_start(zend_jit_ctx *jit, int b)
{
zend_basic_block *bb;
int i, n, *p, pred;
ir_ref ref, bb_start;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
ZEND_ASSERT(b < jit->ssa->cfg.blocks_count);
bb = &jit->ssa->cfg.blocks[b];
ZEND_ASSERT((bb->flags & ZEND_BB_REACHABLE) != 0);
n = bb->predecessors_count;
if (n == 0) {
/* pass */
ZEND_ASSERT(jit->ctx.control);
#if ZEND_DEBUG
ref = jit->ctx.control;
ir_insn *insn = &jit->ctx.ir_base[ref];
while (insn->op >= IR_CALL && insn->op <= IR_TRAP) {
ref = insn->op1;
insn = &jit->ctx.ir_base[ref];
}
ZEND_ASSERT(insn->op == IR_START);
ZEND_ASSERT(ref == 1);
#endif
bb_start = 1;
if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) {
/* prvent END/BEGIN merging */
jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END());
bb_start = jit->ctx.control;
}
} else if (n == 1) {
ZEND_ASSERT(!jit->ctx.control);
pred = jit->ssa->cfg.predecessors[bb->predecessor_offset];
ref = jit->bb_edges[jit->bb_predecessors[b]];
if (ref == IR_UNUSED) {
if (!jit->ctx.control) {
ir_BEGIN(IR_UNUSED); /* unreachable block */
}
} else {
ir_op op = jit->ctx.ir_base[ref].op;
if (op == IR_IF) {
if (!jit->ctx.control) {
jit_IF_TRUE_FALSE_ex(jit, ref, b);
} else {
ir_ref entry_path = ir_END();
jit_IF_TRUE_FALSE_ex(jit, ref, b);
ir_MERGE_WITH(entry_path);
}
} else if (op == IR_SWITCH) {
zend_jit_case_start(jit, pred, b, ref);
} else {
if (!jit->ctx.control) {
ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY)
&& (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) {
/* prvent END/BEGIN merging */
jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref);
} else {
ir_BEGIN(ref);
}
} else {
ir_MERGE_WITH(ref);
}
}
}
bb_start = jit->ctx.control;
} else {
int forward_edges_count = 0;
int back_edges_count = 0;
ir_ref *pred_refs;
ir_ref entry_path = IR_UNUSED;
ALLOCA_FLAG(use_heap);
ZEND_ASSERT(!jit->ctx.control);
if (jit->ctx.control) {
entry_path = ir_END();
}
pred_refs = (ir_ref *)do_alloca(sizeof(ir_ref) * n, use_heap);
for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) {
pred = *p;
if (jit->bb_start_ref[pred]) {
/* forward edge */
forward_edges_count++;
ref = jit->bb_edges[jit->bb_predecessors[b] + i];
if (ref == IR_UNUSED) {
/* dead edge */
pred_refs[i] = IR_UNUSED;
} else {
ir_op op = jit->ctx.ir_base[ref].op;
if (op == IR_IF) {
jit_IF_TRUE_FALSE_ex(jit, ref, b);
pred_refs[i] = ir_END();
} else if (op == IR_SWITCH) {
zend_jit_case_start(jit, pred, b, ref);
pred_refs[i] = ir_END();
} else {
ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN);
pred_refs[i] = ref;
}
}
} else {
/* backward edge */
back_edges_count++;
pred_refs[i] = IR_UNUSED;
}
}
if (bb->flags & ZEND_BB_LOOP_HEADER) {
ZEND_ASSERT(back_edges_count != 0);
ZEND_ASSERT(forward_edges_count != 0);
ir_MERGE_N(n, pred_refs);
jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN;
bb_start = jit->ctx.control;
if (entry_path) {
ir_MERGE_WITH(entry_path);
}
} else {
// ZEND_ASSERT(back_edges_count != 0);
/* edges from exceptional blocks may be counted as back edges */
ir_MERGE_N(n, pred_refs);
bb_start = jit->ctx.control;
if (entry_path) {
ir_MERGE_WITH(entry_path);
}
}
free_alloca(pred_refs, use_heap);
}
jit->b = b;
jit->bb_start_ref[b] = bb_start;
if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_cse_limit)) {
jit->ctx.fold_cse_limit = bb_start;
}
return 1;
}
static int zend_jit_bb_end(zend_jit_ctx *jit, int b)
{
int succ;
zend_basic_block *bb;
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
if (jit->b != b) {
return 1;
}
bb = &jit->ssa->cfg.blocks[b];
ZEND_ASSERT(bb->successors_count != 0);
if (bb->successors_count == 1) {
succ = bb->successors[0];
} else {
const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1];
/* Use only the following successor of SWITCH and FE_RESET_R */
ZEND_ASSERT(opline->opcode == ZEND_SWITCH_LONG
|| opline->opcode == ZEND_SWITCH_STRING
|| opline->opcode == ZEND_MATCH
|| opline->opcode == ZEND_FE_RESET_R);
succ = b + 1;
}
_zend_jit_add_predecessor_ref(jit, succ, b, ir_END());
jit->b = -1;
return 1;
}
static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline)
{
ir_ref ref;
#if 1
if (GCC_GLOBAL_REGS) {
ref = jit_IP32(jit);
} else {
ref = ir_LOAD_U32(jit_EX(opline));
}
ref = ir_CMP_OP(op, ref, ir_CONST_U32((uint32_t)(uintptr_t)next_opline));
#else
if (GCC_GLOBAL_REGS) {
ref = jit_IP(jit);
} else {
ref = ir_LOAD_A(jit_EX(opline));
}
ref = ir_CMP_OP(op, ref, ir_CONST_ADDR(next_opline));
#endif
return ref;
}
static int zend_jit_jmp_frameless(
zend_jit_ctx *jit,
const zend_op *opline,
const void *exit_addr,
zend_jmp_fl_result guard
) {
ir_ref ref, if_ref, cache_result, function_result, phi_result, cache_slot_ref;
zend_basic_block *bb;
// JIT: CACHED_PTR(opline->extended_value)
cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
cache_result = ir_LOAD_L(cache_slot_ref);
// JIT: if (UNEXPECTED(!result))
if_ref = ir_IF(cache_result);
ir_IF_FALSE_cold(if_ref);
zval *func_name_zv = RT_CONSTANT(opline, opline->op1);
function_result = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_jit_jmp_frameless_helper),
ir_CONST_ADDR(func_name_zv),
cache_slot_ref);
ir_MERGE_WITH_EMPTY_TRUE(if_ref);
phi_result = ir_PHI_2(IR_LONG, function_result, cache_result);
if (exit_addr) {
ir_GUARD(ir_EQ(phi_result, ir_CONST_LONG(guard)), ir_CONST_ADDR(exit_addr));
} else {
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
// JIT: if (result == ZEND_JMP_FL_HIT)
ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
jit->b = -1;
}
return 1;
}
static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block)
{
ir_ref ref;
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
if (bb->successors[0] == bb->successors[1]) {
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
jit->b = -1;
zend_jit_set_last_valid_opline(jit, next_opline);
return 1;
}
ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
jit->b = -1;
zend_jit_set_last_valid_opline(jit, next_opline);
return 1;
}
static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *next_opline, uint32_t var)
{
ir_ref ref;
ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, IR_EQ, next_opline)), ir_CONST_U32(IS_FALSE));
// EX_VAR(var) = ...
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref);
zend_jit_reset_last_valid_opline(jit);
return zend_jit_set_ip(jit, next_opline - 1);
}
/* PHP JIT handlers */
static void zend_jit_check_exception(zend_jit_ctx *jit)
{
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline)
{
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit,
(opline->result_type & (IS_TMP_VAR|IS_VAR)) ? jit_stub_exception_handler_undef : jit_stub_exception_handler));
}
static void zend_jit_type_check_undef(zend_jit_ctx *jit,
ir_ref type,
uint32_t var,
const zend_op *opline,
bool check_exception,
bool in_cold_path,
bool undef_result)
{
ir_ref if_def = ir_IF(type);
if (!in_cold_path) {
ir_IF_FALSE_cold(if_def);
} else {
ir_IF_FALSE(if_def);
}
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
if (check_exception) {
if (undef_result) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
ir_MERGE_WITH_EMPTY_TRUE(if_def);
}
static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit,
ir_ref ref,
uint32_t var,
const zend_op *opline,
bool check_exception)
{
ir_ref if_def, ref2;
if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
ir_IF_FALSE_cold(if_def);
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var));
if (check_exception) {
zend_jit_check_exception(jit);
}
ref2 = jit_EG(uninitialized_zval);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
return ir_PHI_2(IR_ADDR, ref2, ref);
}
static void zend_jit_recv_entry(zend_jit_ctx *jit, int b)
{
zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
int pred;
ir_ref ref;
ZEND_ASSERT(bb->predecessors_count > 0);
pred = jit->bb_predecessors[b];
ref = jit->bb_edges[pred];
ZEND_ASSERT(ref);
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END);
/* Insert a MERGE block with additional ENTRY input between predecessor and this one */
ir_ENTRY(ref, bb->start);
if (!GCC_GLOBAL_REGS) {
/* 2 is hardcoded reference to IR_PARAM */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
jit_STORE_FP(jit, 2);
}
ir_MERGE_WITH(ref);
jit->bb_edges[pred] = ir_END();
}
static void zend_jit_osr_entry(zend_jit_ctx *jit, int b)
{
zend_basic_block *bb = &jit->ssa->cfg.blocks[b];
ir_ref ref = ir_END();
/* Insert a MERGE block with additional ENTRY input between predecessor and this one */
ir_ENTRY(ref, bb->start);
if (!GCC_GLOBAL_REGS) {
/* 2 is hardcoded reference to IR_PARAM */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
jit_STORE_FP(jit, 2);
}
ir_MERGE_WITH(ref);
}
static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label)
{
ir_ENTRY(src, label);
if (!GCC_GLOBAL_REGS) {
/* 2 is hardcoded reference to IR_PARAM */
ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM);
ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1);
jit_STORE_FP(jit, 2);
}
return ir_END();
}
static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw)
{
ir_ref ref;
const void *handler;
zend_jit_set_ip(jit, opline);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
handler = zend_get_opcode_handler_func(opline);
} else {
handler = opline->handler;
}
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
ref = jit_FP(jit);
ref = ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(handler), ref);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
/* Skip the following OP_DATA */
switch (opline->opcode) {
case ZEND_ASSIGN_DIM:
case ZEND_ASSIGN_OBJ:
case ZEND_ASSIGN_STATIC_PROP:
case ZEND_ASSIGN_DIM_OP:
case ZEND_ASSIGN_OBJ_OP:
case ZEND_ASSIGN_STATIC_PROP_OP:
case ZEND_ASSIGN_STATIC_PROP_REF:
case ZEND_ASSIGN_OBJ_REF:
zend_jit_set_last_valid_opline(jit, opline + 2);
break;
default:
zend_jit_set_last_valid_opline(jit, opline + 1);
break;
}
return 1;
}
static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline)
{
const void *handler;
ir_ref ref;
zend_basic_block *bb;
zend_jit_set_ip(jit, opline);
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_RETURN) {
/* Use inlined HYBRID VM handler */
handler = opline->handler;
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
handler = zend_get_opcode_handler_func(opline);
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
ref = ir_LOAD_A(jit_IP(jit));
ir_TAILCALL(IR_VOID, ref);
}
} else {
handler = opline->handler;
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler));
} else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)
&& (opline->opcode == ZEND_CATCH
|| opline->opcode == ZEND_FAST_CALL
|| opline->opcode == ZEND_FAST_RET
|| opline->opcode == ZEND_MATCH_ERROR
|| opline->opcode == ZEND_THROW
|| opline->opcode == ZEND_VERIFY_NEVER_TYPE)) {
ref = jit_FP(jit);
ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
ir_RETURN(ir_CONST_I32(1));
} else {
ref = jit_FP(jit);
ir_TAILCALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
}
}
if (jit->b >= 0) {
bb = &jit->ssa->cfg.blocks[jit->b];
if (bb->successors_count > 0
&& (opline->opcode == ZEND_DO_FCALL
|| opline->opcode == ZEND_DO_UCALL
|| opline->opcode == ZEND_DO_FCALL_BY_NAME
|| opline->opcode == ZEND_INCLUDE_OR_EVAL
|| opline->opcode == ZEND_GENERATOR_CREATE
|| opline->opcode == ZEND_YIELD
|| opline->opcode == ZEND_YIELD_FROM
|| opline->opcode == ZEND_FAST_CALL)) {
/* Add a fake control edge from UNREACHABLE to the following ENTRY */
int succ;
if (bb->successors_count == 1) {
succ = bb->successors[0];
ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
} else {
/* Use only the following successor of FAST_CALL */
ZEND_ASSERT(opline->opcode == ZEND_FAST_CALL);
succ = jit->b + 1;
/* we need an entry */
jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY;
}
ref = jit->ctx.insns_count - 1;
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN);
ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
_zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
}
jit->b = -1;
zend_jit_reset_last_valid_opline(jit);
}
return 1;
}
static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block)
{
return zend_jit_tail_handler(jit, opline);
}
static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
{
ZEND_ASSERT(Z_MODE(src) == IS_REG);
ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
ZEND_ASSERT(Z_MODE(src) == IS_REG);
ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src));
if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
} else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG) {
/* invalidate memory type */
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src));
if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
} else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE) {
/* invalidate memory type */
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN;
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
ZEND_ASSERT(Z_MODE(dst) == IS_REG);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src));
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src));
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type)
{
zend_jit_addr src = ZEND_ADDR_REG(ssa_var);
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
return zend_jit_spill_store(jit, src, dst, info, set_type);
}
static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_LVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_DVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg)
{
ir_ref ref = jit->ctx.control;
ir_insn *insn;
while (1) {
insn = &jit->ctx.ir_base[ref];
if (insn->op == IR_RLOAD && insn->op2 == reg) {
ZEND_ASSERT(insn->type == type);
return ref;
} else if (insn->op == IR_START) {
break;
}
ref = insn->op1;
}
return ir_RLOAD(type, reg);
}
static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ir_ref src = ir_CONST_LONG(val);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
}
jit_set_Z_LVAL(jit, dst, src);
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
return 1;
}
static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ir_ref src = ir_CONST_DOUBLE(val);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
}
jit_set_Z_DVAL(jit, dst, src);
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
return 1;
}
static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ZEND_ASSERT(type <= IS_DOUBLE);
jit_set_Z_TYPE_INFO(jit, dst, type);
return 1;
}
static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, bool set_type)
{
zend_jit_addr src;
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
ir_type type;
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
type = IR_LONG;
src = zend_jit_deopt_rload(jit, type, reg);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else if (!in_mem) {
jit_set_Z_LVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
type = IR_DOUBLE;
src = zend_jit_deopt_rload(jit, type, reg);
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else if (!in_mem) {
jit_set_Z_DVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t offset, bool set_type)
{
zend_jit_addr src;
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
src = ir_LOAD_L(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else {
jit_set_Z_LVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_LONG);
}
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
src = ir_LOAD_D(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset));
if (jit->ra && jit->ra[var].ref == IR_NULL) {
zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src);
} else {
jit_set_Z_DVAL(jit, dst, src);
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE);
}
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
jit_set_Z_TYPE_INFO(jit, dst, type);
return 1;
}
static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr)
{
ir_ref if_refcounted, end1;
if_refcounted = jit_if_REFCOUNTED(jit, var_addr);
ir_IF_FALSE(if_refcounted);
end1 = ir_END();
ir_IF_TRUE(if_refcounted);
jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr));
ir_MERGE_WITH(end1);
return 1;
}
static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
return zend_jit_spill_store(jit, src, dst, info, 1);
}
return 1;
}
static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
bool set_type = 1;
if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
(old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
if (JIT_G(current_frame)) {
uint32_t mem_type = STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var));
if (mem_type != IS_UNKNOWN
&& (info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (1 << mem_type)) {
set_type = 0;
}
} else {
set_type = 0;
}
}
}
return zend_jit_spill_store(jit, src, dst, info, set_type);
}
return 1;
}
static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var)
{
zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
zend_jit_addr dst = ZEND_ADDR_REG(ssa_var);
return zend_jit_load_reg(jit, src, dst, info);
}
static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr addr, znode_op op)
{
if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
/* Invalidate operand type to prevent incorrect destuction by exception_handler_free_op1_op2() */
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF);
}
return 1;
}
static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
if (!zend_jit_same_addr(src, dst)) {
if (Z_MODE(src) == IS_REG) {
if (Z_MODE(dst) == IS_REG) {
zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src));
if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(jit, dst, var_addr, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else if (Z_MODE(dst) == IS_MEM_ZVAL) {
if (!Z_LOAD(src) && !Z_STORE(src)) {
if (!zend_jit_spill_store(jit, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(src) == IS_MEM_ZVAL) {
if (Z_MODE(dst) == IS_REG) {
if (!zend_jit_load_reg(jit, src, dst, info)) {
return 0;
}
} else {
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(jit, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
return 1;
}
struct jit_observer_fcall_is_unobserved_data {
ir_ref if_unobserved;
ir_ref ir_end_inputs;
};
static struct jit_observer_fcall_is_unobserved_data jit_observer_fcall_is_unobserved_start(zend_jit_ctx *jit, const zend_function *func, ir_ref *observer_handler, ir_ref rx, ir_ref func_ref) {
ir_ref run_time_cache;
struct jit_observer_fcall_is_unobserved_data data = { .ir_end_inputs = IR_UNUSED };
if (func) {
ZEND_ASSERT((func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0);
} else {
// JIT: if (function->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) {
ZEND_ASSERT(rx != IR_UNUSED);
ir_ref if_trampoline_or_generator = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)));
ir_IF_TRUE(if_trampoline_or_generator);
ir_END_list(data.ir_end_inputs);
ir_IF_FALSE(if_trampoline_or_generator);
}
if (func && (func->common.fn_flags & ZEND_ACC_CLOSURE) == 0 && ZEND_MAP_PTR_IS_OFFSET(func->common.run_time_cache)) {
// JIT: ZEND_MAP_PTR_GET_IMM(func->common.runtime_cache)
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)), (uintptr_t)ZEND_MAP_PTR(func->common.run_time_cache)));
#ifndef ZTS
} else if (func && rx == IS_UNUSED) { // happens for internal functions only
ZEND_ASSERT(!ZEND_USER_CODE(func->type));
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_CONST_ADDR(func), offsetof(zend_op_array, run_time_cache__ptr)));
#endif
} else {
// Closures may be duplicated and have a different runtime cache. Use the regular run_time_cache access pattern for these
if (func && ZEND_USER_CODE(func->type)) { // not a closure and definitely not an internal function
run_time_cache = ir_LOAD_A(jit_CALL(rx, run_time_cache));
} else {
// JIT: ZEND_MAP_PTR_GET(func->common.runtime_cache)
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_op_array, run_time_cache__ptr)));
ir_ref if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
ir_IF_TRUE(if_odd);
ir_ref run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
ir_ref if_odd_end = ir_END();
ir_IF_FALSE(if_odd);
// JIT: if (func->common.runtime_cache != NULL) {
ir_ref if_rt_cache = ir_IF(ir_EQ(run_time_cache, IR_NULL));
ir_IF_TRUE(if_rt_cache);
ir_END_list(data.ir_end_inputs);
ir_IF_FALSE(if_rt_cache);
ir_MERGE_WITH(if_odd_end);
run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache, run_time_cache2);
}
}
// JIT: observer_handler = runtime_cache + ZEND_OBSERVER_HANDLE(function)
if (func) {
*observer_handler = ir_ADD_OFFSET(run_time_cache, ZEND_OBSERVER_HANDLE(func) * sizeof(void *));
} else {
// JIT: (func->type == ZEND_INTERNAL_FUNCTION ? zend_observer_fcall_internal_function_extension : zend_observer_fcall_op_array_extension) * sizeof(void *)
ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
ir_ref if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(ZEND_INTERNAL_FUNCTION)));
ir_IF_TRUE(if_internal_func);
ir_ref observer_handler_internal = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_internal_function_extension * sizeof(void *));
ir_ref if_internal_func_end = ir_END();
ir_IF_FALSE(if_internal_func);
ir_ref observer_handler_user = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_op_array_extension * sizeof(void *));
ir_MERGE_WITH(if_internal_func_end);
*observer_handler = ir_PHI_2(IR_ADDR, observer_handler_internal, observer_handler_user);
}
// JIT: if (*observer_handler == ZEND_OBSERVER_NONE_OBSERVED) {
data.if_unobserved = ir_IF(ir_EQ(ir_LOAD_A(*observer_handler), ir_CONST_ADDR(ZEND_OBSERVER_NONE_OBSERVED)));
ir_IF_FALSE(data.if_unobserved);
return data;
}
/* For frameless the true branch of if_unobserved is used and this function not called. */
static void jit_observer_fcall_is_unobserved_end(zend_jit_ctx *jit, struct jit_observer_fcall_is_unobserved_data *data) {
ir_END_list(data->ir_end_inputs);
ir_IF_TRUE(data->if_unobserved);
ir_END_list(data->ir_end_inputs);
ir_MERGE_list(data->ir_end_inputs);
}
static void jit_observer_fcall_begin(zend_jit_ctx *jit, ir_ref rx, ir_ref observer_handler) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin_prechecked), rx, observer_handler);
}
static void jit_observer_fcall_end(zend_jit_ctx *jit, ir_ref rx, ir_ref res_ref) {
// JIT: if (execute_data == EG(current_observed_frame)) {
ir_ref has_end_observer = ir_IF(ir_EQ(rx, ir_LOAD_A(jit_EG(current_observed_frame))));
ir_IF_TRUE(has_end_observer);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end_prechecked),
rx, res_ref);
ir_MERGE_WITH_EMPTY_FALSE(has_end_observer);
}
static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
ir_ref if_long = IR_UNUSED;
ir_ref op1_lval_ref = IR_UNUSED;
ir_ref ref;
ir_op op;
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_long);
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
jit_set_Z_LVAL(jit, res_addr, op1_lval_ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL
&& Z_MODE(op1_addr) == IS_REG
&& !Z_LOAD(op1_addr)
&& !Z_STORE(op1_addr)) {
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
op = may_overflow ? IR_ADD_OV : IR_ADD;
} else {
op = may_overflow ? IR_SUB_OV : IR_SUB;
}
if (!op1_lval_ref) {
op1_lval_ref = jit_Z_LVAL(jit, op1_addr);
}
ref = ir_BINARY_OP_L(op, op1_lval_ref, ir_CONST_LONG(1));
if (op1_def_info & MAY_BE_LONG) {
jit_set_Z_LVAL(jit, op1_def_addr, ref);
}
if (may_overflow &&
(((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) ||
((opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD))))) {
int32_t exit_point;
const void *exit_addr;
zend_jit_trace_stack *stack;
uint32_t old_op1_info, old_res_info = 0;
stack = JIT_G(current_frame)->stack;
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
}
if (opline->result_type != IS_UNUSED) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
if (opline->opcode == ZEND_PRE_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else if (opline->opcode == ZEND_PRE_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else if (opline->opcode == ZEND_POST_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MAX));
} else if (opline->opcode == ZEND_POST_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MIN));
}
}
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
if (opline->result_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
} else if (may_overflow) {
ir_ref if_overflow;
ir_ref merge_inputs = IR_UNUSED;
if (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))
|| (opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
int32_t exit_point;
const void *exit_addr;
zend_jit_trace_stack *stack;
uint32_t old_res_info = 0;
stack = JIT_G(current_frame)->stack;
if (opline->result_type != IS_UNUSED) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
}
}
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE_cold(if_overflow);
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
ir_IF_TRUE(if_overflow);
} else {
ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
}
if (opline->result_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
} else {
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE(if_overflow);
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
ir_END_list(merge_inputs);
/* overflow => cold path */
ir_IF_TRUE_cold(if_overflow);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (Z_MODE(op1_def_addr) == IS_REG) {
jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
}
} else {
if (Z_MODE(op1_def_addr) == IS_REG) {
jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE);
}
}
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
if (opline->opcode == ZEND_PRE_INC) {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
} else {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
}
if (merge_inputs) {
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
}
} else {
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
ir_ref merge_inputs = ir_END();
/* !is_long => cold path */
ir_IF_FALSE_cold(if_long);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
jit_SET_EX_OPLINE(jit, opline);
if (op1_info & MAY_BE_UNDEF) {
ir_ref if_def;
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
op1_info |= MAY_BE_NULL;
}
ref = jit_ZVAL_ADDR(jit, op1_addr);
if (op1_info & MAY_BE_REF) {
ir_ref if_ref, if_typed, func, ref2, arg2;
if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_TRUE(if_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
if_typed = jit_if_TYPED_REF(jit, ref2);
ir_IF_TRUE(if_typed);
if (RETURN_VALUE_USED(opline)) {
ZEND_ASSERT(Z_MODE(res_addr) != IS_REG);
arg2 = jit_ZVAL_ADDR(jit, res_addr);
} else {
arg2 = IR_NULL;
}
if (opline->opcode == ZEND_PRE_INC) {
func = ir_CONST_FC_FUNC(zend_jit_pre_inc_typed_ref);
} else if (opline->opcode == ZEND_PRE_DEC) {
func = ir_CONST_FC_FUNC(zend_jit_pre_dec_typed_ref);
} else if (opline->opcode == ZEND_POST_INC) {
func = ir_CONST_FC_FUNC(zend_jit_post_inc_typed_ref);
} else if (opline->opcode == ZEND_POST_DEC) {
func = ir_CONST_FC_FUNC(zend_jit_post_dec_typed_ref);
} else {
ZEND_UNREACHABLE();
}
ir_CALL_2(IR_VOID, func, ref2, arg2);
zend_jit_check_exception(jit);
ir_END_list(merge_inputs);
ir_IF_FALSE(if_typed);
ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
jit_ZVAL_COPY(jit,
res_addr,
res_use_info,
ZEND_ADDR_REF_ZVAL(ref), op1_info, 1);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc), ref, arg2);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function), ref);
}
} else {
if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec), ref, arg2);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function), ref);
}
}
if (may_throw) {
zend_jit_check_exception(jit);
}
} else {
ref = jit_Z_DVAL(jit, op1_addr);
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
jit_set_Z_DVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
op = IR_ADD;
} else {
op = IR_SUB;
}
ref = ir_BINARY_OP_D(op, ref, ir_CONST_DOUBLE(1.0));
jit_set_Z_DVAL(jit, op1_def_addr, ref);
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
jit_set_Z_DVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
}
if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
static int zend_jit_math_long_long(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow)
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_op op;
ir_ref op1, op2, ref, if_overflow = IR_UNUSED;
if (opcode == ZEND_ADD) {
op = may_overflow ? IR_ADD_OV : IR_ADD;
} else if (opcode == ZEND_SUB) {
op = may_overflow ? IR_SUB_OV : IR_SUB;
} else if (opcode == ZEND_MUL) {
op = may_overflow ? IR_MUL_OV : IR_MUL;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
ref = ir_BINARY_OP_L(op, op1, op2);
if (may_overflow) {
if (res_info & MAY_BE_GUARD) {
if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_res_info;
int32_t exit_point;
const void *exit_addr;
if (opline->opcode == ZEND_ADD
&& Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
} else if (opline->opcode == ZEND_SUB
&& Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
} else {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
}
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
may_overflow = 0;
} else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr));
} else {
ZEND_UNREACHABLE();
}
} else {
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE(if_overflow);
}
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
}
}
if (may_overflow) {
ir_ref fast_path = IR_UNUSED;
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
fast_path = ir_END();
ir_IF_TRUE_cold(if_overflow);
}
if (opcode == ZEND_ADD) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
ir_MERGE_WITH(fast_path);
}
return 1;
}
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) {
if (Z_MODE(res_addr) == IS_REG) {
jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0));
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
ir_MERGE_WITH(fast_path);
}
return 1;
}
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else {
ZEND_UNREACHABLE();
}
#if 1
/* reload */
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
#endif
#if 1
/* disable CSE */
ir_ref old_cse_limit = jit->ctx.fold_cse_limit;
jit->ctx.fold_cse_limit = 0x7fffffff;
#endif
op1 = ir_INT2D(op1);
op2 = ir_INT2D(op2);
#if 1
jit->ctx.fold_cse_limit = old_cse_limit;
#endif
ref = ir_BINARY_OP_D(op, op1, op2);
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) {
ir_MERGE_WITH(fast_path);
}
}
return 1;
}
static int zend_jit_math_long_double(zend_jit_ctx *jit,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
ir_op op;
ir_ref op1, op2, ref;
if (opcode == ZEND_ADD) {
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else if (opcode == ZEND_DIV) {
op = IR_DIV;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = jit_Z_DVAL(jit, op2_addr);
ref = ir_BINARY_OP_D(op, ir_INT2D(op1), op2);
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
return 1;
}
static int zend_jit_math_double_long(zend_jit_ctx *jit,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
ir_op op;
ir_ref op1, op2, ref;
if (opcode == ZEND_ADD) {
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else if (opcode == ZEND_DIV) {
op = IR_DIV;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_DVAL(jit, op1_addr);
op2 = jit_Z_LVAL(jit, op2_addr);
ref = ir_BINARY_OP_D(op, op1, ir_INT2D(op2));
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
}
return 1;
}
static int zend_jit_math_double_double(zend_jit_ctx *jit,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_op op;
ir_ref op1, op2, ref;
if (opcode == ZEND_ADD) {
op = IR_ADD;
} else if (opcode == ZEND_SUB) {
op = IR_SUB;
} else if (opcode == ZEND_MUL) {
op = IR_MUL;
} else if (opcode == ZEND_DIV) {
op = IR_DIV;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_DVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr);
ref = ir_BINARY_OP_D(op, op1, op2);
jit_set_Z_DVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
}
return 1;
}
static int zend_jit_math_helper(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow,
int may_throw)
{
ir_ref if_op1_long = IR_UNUSED;
ir_ref if_op1_double = IR_UNUSED;
ir_ref if_op2_double = IR_UNUSED;
ir_ref if_op1_long_op2_long = IR_UNUSED;
ir_ref if_op1_long_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_long = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_refs *end_inputs;
ir_refs *res_inputs;
ir_refs_init(end_inputs, 6);
ir_refs_init(res_inputs, 6);
if (Z_MODE(op1_addr) == IS_REG) {
if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op1_addr);
}
} else if (Z_MODE(op2_addr) == IS_REG) {
if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op2_addr);
}
}
if (Z_MODE(res_addr) == IS_REG) {
jit->delay_var = Z_SSA_VAR(res_addr);
jit->delay_refs = res_inputs;
}
if ((res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG) && (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_op1_long);
}
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_TRUE(if_op1_long_op2_long);
}
if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_long) {
ir_IF_FALSE_cold(if_op1_long);
ir_END_list(slow_inputs);
}
if (if_op1_long_op2_long) {
ir_IF_FALSE_cold(if_op1_long_op2_long);
ir_END_list(slow_inputs);
}
} else if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_op1_long);
}
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_long);
if (op2_info & MAY_BE_DOUBLE) {
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else {
ir_END_list(slow_inputs);
}
ir_IF_TRUE(if_op1_long_op2_long);
}
if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_long) {
ir_IF_FALSE_cold(if_op1_long);
}
if (op1_info & MAY_BE_DOUBLE) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops) {
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if (if_op1_long) {
ir_END_list(slow_inputs);
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op2_double);
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
ir_refs_add(end_inputs, ir_END());
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
ir_ref func, arg1, arg2, arg3;
if (slow_inputs) {
ir_MERGE_list(slow_inputs);
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
if (Z_MODE(res_addr) == IS_REG) {
arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
} else {
arg1 = jit_ZVAL_ADDR(jit, res_addr);
}
arg2 = jit_ZVAL_ADDR(jit, op1_addr);
arg3 = jit_ZVAL_ADDR(jit, op2_addr);
jit_SET_EX_OPLINE(jit, opline);
if (opcode == ZEND_ADD) {
func = ir_CONST_FC_FUNC(add_function);
} else if (opcode == ZEND_SUB) {
func = ir_CONST_FC_FUNC(sub_function);
} else if (opcode == ZEND_MUL) {
func = ir_CONST_FC_FUNC(mul_function);
} else if (opcode == ZEND_DIV) {
func = ir_CONST_FC_FUNC(div_function);
} else {
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_refs_add(end_inputs, ir_END());
}
if (end_inputs->count) {
ir_MERGE_N(end_inputs->count, end_inputs->refs);
}
if (Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(end_inputs->count == res_inputs->count);
jit->delay_var = -1;
jit->delay_refs = NULL;
if (res_inputs->count == 1) {
zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
} else {
ir_ref phi = ir_PHI_N((res_info & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, res_inputs->count, res_inputs->refs);
zend_jit_def_reg(jit, res_addr, phi);
}
}
return 1;
}
static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
{
ir_ref ref;
ir_ref arg1 = jit_Z_PTR(jit, op1_addr);
ir_ref arg2 = jit_Z_PTR(jit, op2_addr);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_add_arrays_helper), arg1, arg2);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX);
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
return 1;
}
static int zend_jit_long_math_helper(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
zend_ssa_range *op1_range,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_throw)
{
ir_ref ref = IR_UNUSED;
ir_ref if_long1 = IR_UNUSED;
ir_ref if_long2 = IR_UNUSED;
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
ir_refs *res_inputs;
ir_refs_init(res_inputs, 2);
if (Z_MODE(op1_addr) == IS_REG
&& Z_LOAD(op1_addr)
&& jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op1_addr);
}
if (Z_MODE(op2_addr) == IS_REG
&& Z_LOAD(op2_addr)
&& jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op2_addr);
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_long1);
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_TRUE(if_long2);
}
if (opcode == ZEND_SL) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
ref = ir_CONST_LONG(0);
} else {
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
if (Z_MODE(res_addr) == IS_REG) {
ref = ir_CONST_LONG(0); // dead code
}
}
} else {
ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
}
} else {
ref = jit_Z_LVAL(jit, op2_addr);
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
ir_ref if_wrong, cold_path, ref2, if_ok;
ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
ir_IF_TRUE_cold(if_wrong);
if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
ir_IF_FALSE(if_ok);
jit_SET_EX_OPLINE(jit, opline);
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
ir_IF_TRUE(if_ok);
ref2 = ir_CONST_LONG(0);
cold_path = ir_END();
ir_IF_FALSE(if_wrong);
ref = ir_SHL_L(op1_ref, ref);
ir_MERGE_WITH(cold_path);
ref = ir_PHI_2(IR_LONG, ref, ref2);
} else {
ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref);
}
}
} else if (opcode == ZEND_SR) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
ref = ir_SAR_L(
jit_Z_LVAL(jit, op1_addr),
ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1));
} else {
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift));
if (Z_MODE(res_addr) == IS_REG) {
ref = ir_CONST_LONG(0); // dead code
}
}
} else {
ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
}
} else {
ref = jit_Z_LVAL(jit, op2_addr);
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
ir_ref if_wrong, cold_path, ref2, if_ok;
if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)));
ir_IF_TRUE_cold(if_wrong);
if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0)));
ir_IF_FALSE(if_ok);
jit_SET_EX_OPLINE(jit, opline);
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift));
ir_IF_TRUE(if_ok);
ref2 = ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1);
cold_path = ir_END();
ir_IF_FALSE(if_wrong);
ir_MERGE_WITH(cold_path);
ref = ir_PHI_2(IR_LONG, ref, ref2);
}
ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref);
}
} else if (opcode == ZEND_MOD) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (op2_lval == 0) {
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
if (Z_MODE(res_addr) == IS_REG) {
ref = ir_CONST_LONG(0); // dead code
}
} else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1));
} else {
ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval));
}
} else {
ir_ref zero_path = 0;
ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr);
ref = jit_Z_LVAL(jit, op2_addr);
if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
ir_ref if_ok = ir_IF(ref);
ir_IF_FALSE(if_ok);
jit_SET_EX_OPLINE(jit, opline);
zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero));
ir_IF_TRUE(if_ok);
}
/* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
ir_ref if_minus_one = ir_IF(ir_EQ(ref, ir_CONST_LONG(-1)));
ir_IF_TRUE_cold(if_minus_one);
zero_path = ir_END();
ir_IF_FALSE(if_minus_one);
}
ref = ir_MOD_L(op1_ref, ref);
if (zero_path) {
ir_MERGE_WITH(zero_path);
ref = ir_PHI_2(IR_LONG, ref, ir_CONST_LONG(0));
}
}
} else {
ir_op op;
ir_ref op1, op2;
if (opcode == ZEND_BW_OR) {
op = IR_OR;
} else if (opcode == ZEND_BW_AND) {
op = IR_AND;
} else if (opcode == ZEND_BW_XOR) {
op = IR_XOR;
} else {
ZEND_UNREACHABLE();
}
op1 = jit_Z_LVAL(jit, op1_addr);
op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr);
ref = ir_BINARY_OP_L(op, op1, op2);
}
if (ref) {
if (Z_MODE(res_addr) == IS_REG
&& ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))
|| (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
jit->delay_var = Z_SSA_VAR(res_addr);
jit->delay_refs = res_inputs;
}
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) != IS_REG) {
if (!zend_jit_same_addr(op1_addr, res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
}
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
ir_ref fast_path = ir_END();
ir_ref func, arg1, arg2, arg3;
if (if_long2 && if_long1) {
ir_ref ref;
ir_IF_FALSE_cold(if_long2);
ref = ir_END();
ir_IF_FALSE_cold(if_long1);
ir_MERGE_2(ref, ir_END());
} else if (if_long1) {
ir_IF_FALSE_cold(if_long1);
} else if (if_long2) {
ir_IF_FALSE_cold(if_long2);
}
if (op1_info & MAY_BE_UNDEF) {
ir_ref if_def, ref, ref2;
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
ref2 = jit_EG(uninitialized_zval);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op2_info & MAY_BE_UNDEF) {
ir_ref if_def, ref, ref2;
ref = jit_ZVAL_ADDR(jit, op2_addr);
if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var)));
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
ref2 = jit_EG(uninitialized_zval);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
op2_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
if (Z_MODE(res_addr) == IS_REG) {
arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var));
} else {
arg1 = jit_ZVAL_ADDR(jit, res_addr);
}
arg2 = jit_ZVAL_ADDR(jit, op1_addr);
arg3 = jit_ZVAL_ADDR(jit, op2_addr);
jit_SET_EX_OPLINE(jit, opline);
if (opcode == ZEND_BW_OR) {
func = ir_CONST_FC_FUNC(bitwise_or_function);
} else if (opcode == ZEND_BW_AND) {
func = ir_CONST_FC_FUNC(bitwise_and_function);
} else if (opcode == ZEND_BW_XOR) {
func = ir_CONST_FC_FUNC(bitwise_xor_function);
} else if (opcode == ZEND_SL) {
func = ir_CONST_FC_FUNC(shift_left_function);
} else if (opcode == ZEND_SR) {
func = ir_CONST_FC_FUNC(shift_right_function);
} else if (opcode == ZEND_MOD) {
func = ir_CONST_FC_FUNC(mod_function);
} else {
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_VOID, func, arg1, arg2, arg3);
if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
/* compound assignment may decrement "op2" refcount */
op2_info |= MAY_BE_RC1;
}
jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_MERGE_2(fast_path, ir_END());
if (Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(res_inputs->count == 2);
jit->delay_var = -1;
jit->delay_refs = NULL;
if (res_inputs->count == 1) {
zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]);
} else {
ir_ref phi = ir_PHI_N(IR_LONG, res_inputs->count, res_inputs->refs);
zend_jit_def_reg(jit, res_addr, phi);
}
}
}
return 1;
}
static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
if (!zend_jit_long_math_helper(jit, opline, opline->opcode,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_concat_helper(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
int may_throw)
{
ir_ref if_op1_string = IR_UNUSED;
ir_ref if_op2_string = IR_UNUSED;
ir_ref fast_path = IR_UNUSED;
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_op1_string);
}
if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
ir_IF_TRUE(if_op2_string);
}
if (zend_jit_same_addr(op1_addr, res_addr)) {
ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_assign_concat_helper), arg1, arg2);
/* concatenation with itself may reduce refcount */
op2_info |= MAY_BE_RC1;
} else {
ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
if (op1_type == IS_CV || op1_type == IS_CONST) {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_helper), arg1, arg2, arg3);
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_tmp_helper), arg1, arg2, arg3);
}
}
/* concatenation with empty string may increase refcount */
op2_info |= MAY_BE_RCN;
jit_FREE_OP(jit, op2_type, op2, op2_info, opline);
if (if_op1_string || if_op2_string) {
fast_path = ir_END();
}
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
(op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
if (if_op1_string && if_op2_string) {
ir_IF_FALSE(if_op1_string);
ir_MERGE_WITH_EMPTY_FALSE(if_op2_string);
} else if (if_op1_string) {
ir_IF_FALSE_cold(if_op1_string);
} else if (if_op2_string) {
ir_IF_FALSE_cold(if_op2_string);
}
}
ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr);
ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(concat_function), arg1, arg2, arg3);
/* concatenation with empty string may increase refcount */
op1_info |= MAY_BE_RCN;
op2_info |= MAY_BE_RCN;
jit_FREE_OP(jit, op1_type, op1, op1_info, NULL);
jit_FREE_OP(jit, op2_type, op2, op2_info, NULL);
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2));
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(jit, opline);
} else {
zend_jit_check_exception(jit);
}
}
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
ir_MERGE_WITH(fast_path);
}
}
return 1;
}
static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
}
static int zend_jit_assign_op(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_ssa_range *op1_range,
uint32_t op1_def_info,
zend_jit_addr op1_def_addr,
uint32_t op1_mem_info,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
int may_overflow,
int may_throw)
{
int result = 1;
ir_ref slow_path = IR_UNUSED;
ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
if (op1_info & MAY_BE_REF) {
ir_ref ref, ref2, arg2, op1_noref_path;
ir_ref if_op1_ref = IR_UNUSED;
ir_ref if_op1_typed = IR_UNUSED;
binary_op_type binary_op = get_binary_op(opline->extended_value);
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_FALSE(if_op1_ref);
op1_noref_path = ir_END();
ir_IF_TRUE(if_op1_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
if_op1_typed = jit_if_TYPED_REF(jit, ref2);
ir_IF_TRUE_cold(if_op1_typed);
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
jit_SET_EX_OPLINE(jit, opline);
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref_tmp),
ref2, arg2, ir_CONST_FC_FUNC(binary_op));
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
ref2, arg2, ir_CONST_FC_FUNC(binary_op));
}
zend_jit_check_exception(jit);
slow_path = ir_END();
ir_IF_FALSE(if_op1_typed);
ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
ir_MERGE_WITH(op1_noref_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
ZEND_ASSERT(op1_addr == op1_def_addr);
op1_def_addr = op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_overflow, may_throw);
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
result = zend_jit_long_math_helper(jit, opline, opline->extended_value,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_throw);
break;
case ZEND_CONCAT:
result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_def_addr, may_throw);
break;
default:
ZEND_UNREACHABLE();
}
if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
return 0;
}
if (op1_info & MAY_BE_REF) {
ir_MERGE_WITH(slow_path);
}
return result;
}
static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref if_ref, ref2;
if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE)));
ir_IF_TRUE(if_ref);
ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
return ir_PHI_2(IR_ADDR, ref2, ref);
}
static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
{
ir_ref ref = jit_ZVAL_ADDR(jit, addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
return ZEND_ADDR_REF_ZVAL(ref);
}
static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref)
{
ir_ref if_ref, ref2;
if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT)));
ir_IF_TRUE(if_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
return ir_PHI_2(IR_ADDR, ref2, ref);
}
static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr)
{
ir_ref ref = jit_ZVAL_ADDR(jit, addr);
ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref);
return ZEND_ADDR_REF_ZVAL(ref);
}
static int zend_jit_simple_assign(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
bool check_exception)
{
ir_ref end_inputs = IR_UNUSED;
if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
zval *zv = Z_ZV(val_addr);
if (!res_addr) {
jit_ZVAL_COPY_CONST(jit,
var_addr,
var_info, var_def_info,
zv, 1);
} else {
jit_ZVAL_COPY_CONST(jit,
var_addr,
var_info, var_def_info,
zv, 1);
jit_ZVAL_COPY_CONST(jit,
res_addr,
-1, var_def_info,
zv, 1);
}
} else {
if (val_info & MAY_BE_UNDEF) {
ir_ref if_def, ret;
if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL);
if (res_addr) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
jit_SET_EX_OPLINE(jit, opline);
ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL);
// zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var)));
ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
if (check_exception) {
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
}
ir_END_list(end_inputs);
ir_IF_TRUE(if_def);
}
if (val_info & MAY_BE_REF) {
if (val_type == IS_CV) {
ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
val_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_ref ref, type, if_ref, ref2, refcount, if_not_zero;
ref = jit_ZVAL_ADDR(jit, val_addr);
type = jit_Z_TYPE_ref(jit, ref);
if_ref = ir_IF(ir_EQ(type, ir_CONST_U8(IS_REFERENCE)));
ir_IF_TRUE_cold(if_ref);
ref = jit_Z_PTR_ref(jit, ref);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
if (!res_addr) {
jit_ZVAL_COPY(jit,
var_addr,
var_info,
ZEND_ADDR_REF_ZVAL(ref2), val_info, 1);
} else {
jit_ZVAL_COPY_2(jit,
res_addr,
var_addr,
var_info,
ZEND_ADDR_REF_ZVAL(ref2), val_info, 2);
}
refcount = jit_GC_DELREF(jit, ref);
if_not_zero = ir_IF(refcount);
ir_IF_FALSE(if_not_zero);
// TODO: instead of dtor() call and ADDREF above, we may call efree() and move addref at "true" path ???
// This is related to GH-10168 (keep this before GH-10168 is completely closed)
// jit_EFREE(jit, ref, sizeof(zend_reference), NULL, NULL);
jit_ZVAL_DTOR(jit, ref, val_info, opline);
ir_END_list(end_inputs);
ir_IF_TRUE(if_not_zero);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
}
}
if (!res_addr) {
jit_ZVAL_COPY(jit,
var_addr,
var_info,
val_addr, val_info, val_type == IS_CV);
} else {
jit_ZVAL_COPY_2(jit,
res_addr,
var_addr,
var_info,
val_addr, val_info, val_type == IS_CV ? 2 : 1);
}
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr __var_use_addr,
zend_jit_addr var_addr,
uint32_t __var_info,
uint32_t __var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr __res_addr,
bool __check_exception)
{
jit_stub_id func;
ir_ref undef_path = IR_UNUSED;
if (val_info & MAY_BE_UNDEF) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr);
} else {
ir_ref if_def;
ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
if_def = ir_IF(jit_Z_TYPE(jit, val_addr));
ir_IF_FALSE_cold(if_def);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr)));
ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC),
jit_ZVAL_ADDR(jit, var_addr),
jit_EG(uninitialized_zval));
undef_path = ir_END();
ir_IF_TRUE(if_def);
}
}
if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
func = jit_stub_assign_tmp;
} else if (val_type == IS_CONST) {
func = jit_stub_assign_const;
} else if (val_type == IS_TMP_VAR) {
func = jit_stub_assign_tmp;
} else if (val_type == IS_VAR) {
if (!(val_info & MAY_BE_REF)) {
func = jit_stub_assign_tmp;
} else {
func = jit_stub_assign_var;
}
} else if (val_type == IS_CV) {
if (!(val_info & MAY_BE_REF)) {
func = jit_stub_assign_cv_noref;
} else {
func = jit_stub_assign_cv;
}
} else {
ZEND_UNREACHABLE();
}
if (opline) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC),
jit_ZVAL_ADDR(jit, var_addr),
jit_ZVAL_ADDR(jit, val_addr));
if (undef_path) {
ir_MERGE_WITH(undef_path);
}
return 1;
}
static int zend_jit_assign_to_variable(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
zend_jit_addr ref_addr,
bool check_exception)
{
ir_ref if_refcounted = IR_UNUSED;
ir_ref simple_inputs = IR_UNUSED;
bool done = 0;
zend_jit_addr real_res_addr = 0;
ir_refs *end_inputs;
ir_refs *res_inputs;
ir_refs_init(end_inputs, 6);
ir_refs_init(res_inputs, 6);
if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, val_addr);
}
if (Z_MODE(var_addr) == IS_REG) {
jit->delay_var = Z_SSA_VAR(var_addr);
jit->delay_refs = res_inputs;
if (Z_MODE(res_addr) == IS_REG) {
real_res_addr = res_addr;
res_addr = 0;
}
} else if (Z_MODE(res_addr) == IS_REG) {
jit->delay_var = Z_SSA_VAR(res_addr);
jit->delay_refs = res_inputs;
}
if ((var_info & MAY_BE_REF) || ref_addr) {
ir_ref ref = 0, if_ref = 0, ref2, arg2, if_typed, non_ref_path;
uintptr_t func;
if (!ref_addr) {
ref = jit_ZVAL_ADDR(jit, var_use_addr);
if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_TRUE(if_ref);
ref2 = jit_Z_PTR_ref(jit, ref);
} else {
ref2 = jit_ZVAL_ADDR(jit, ref_addr);
}
if_typed = jit_if_TYPED_REF(jit, ref2);
ir_IF_TRUE_cold(if_typed);
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr;
if (opline->opcode == ZEND_ASSIGN_DIM || opline->opcode == ZEND_ASSIGN_OBJ) {
real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
} else {
ZEND_ASSERT(opline->opcode == ZEND_ASSIGN);
real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
}
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!res_addr) {
if (val_type == IS_CONST) {
func = (uintptr_t)zend_jit_assign_const_to_typed_ref;
} else if (val_type == IS_TMP_VAR) {
func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref;
} else if (val_type == IS_VAR) {
func = (uintptr_t)zend_jit_assign_var_to_typed_ref;
} else if (val_type == IS_CV) {
func = (uintptr_t)zend_jit_assign_cv_to_typed_ref;
} else {
ZEND_UNREACHABLE();
}
ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2);
} else {
if (val_type == IS_CONST) {
func = (uintptr_t)zend_jit_assign_const_to_typed_ref2;
} else if (val_type == IS_TMP_VAR) {
func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref2;
} else if (val_type == IS_VAR) {
func = (uintptr_t)zend_jit_assign_var_to_typed_ref2;
} else if (val_type == IS_CV) {
func = (uintptr_t)zend_jit_assign_cv_to_typed_ref2;
} else {
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr));
}
if (check_exception) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
if (!ref_addr) {
ir_IF_FALSE(if_ref);
non_ref_path = ir_END();
ir_IF_FALSE(if_typed);
ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val));
ir_MERGE_WITH(non_ref_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
var_addr = var_use_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_IF_FALSE(if_typed);
}
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
ir_ref ref, counter, if_not_zero;
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(simple_inputs);
ir_IF_TRUE_cold(if_refcounted);
} else if (RC_MAY_BE_1(var_info)) {
done = 1;
}
ref = jit_Z_PTR(jit, var_use_addr);
if (RC_MAY_BE_1(var_info)) {
if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
counter = jit_GC_DELREF(jit, ref);
if_not_zero = ir_IF(counter);
ir_IF_FALSE(if_not_zero);
jit_ZVAL_DTOR(jit, ref, var_info, opline);
if (check_exception) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
ir_IF_TRUE(if_not_zero);
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_FALSE(if_may_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(res_inputs->count > 0);
ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
}
if (check_exception && (val_info & MAY_BE_UNDEF)) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
ir_IF_TRUE(if_may_leak);
}
if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(res_inputs->count > 0);
ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]);
}
if (check_exception && (val_info & MAY_BE_UNDEF)) {
zend_jit_check_exception(jit);
}
ir_refs_add(end_inputs, ir_END());
} else /* if (RC_MAY_BE_N(var_info)) */ {
jit_GC_DELREF(jit, ref);
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref);
ir_IF_FALSE(if_may_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref);
ir_END_list(simple_inputs);
ir_IF_TRUE(if_may_leak);
}
ir_END_list(simple_inputs);
}
}
if (simple_inputs) {
ir_MERGE_list(simple_inputs);
}
if (!done) {
if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, check_exception)) {
return 0;
}
if (end_inputs->count) {
ir_refs_add(end_inputs, ir_END());
}
}
if (end_inputs->count) {
ir_MERGE_N(end_inputs->count, end_inputs->refs);
}
if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) {
ir_ref phi;
ZEND_ASSERT(jit->delay_refs == res_inputs);
ZEND_ASSERT(end_inputs->count == res_inputs->count || (end_inputs->count == 0 && res_inputs->count == 1));
jit->delay_var = -1;
jit->delay_refs = NULL;
if (res_inputs->count == 1) {
phi = res_inputs->refs[0];
} else {
phi = ir_PHI_N((var_def_info & MAY_BE_LONG & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
res_inputs->count, res_inputs->refs);
}
if (Z_MODE(var_addr) == IS_REG) {
if ((var_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || ref_addr) {
phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1);
}
zend_jit_def_reg(jit, var_addr, phi);
if (real_res_addr) {
if (var_def_info & MAY_BE_LONG) {
jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr));
} else {
jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr));
}
}
} else {
zend_jit_def_reg(jit, res_addr, phi);
}
}
return 1;
}
static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
{
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr = op1_def_addr;
}
}
if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 1)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_assign(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_use_addr,
uint32_t op1_def_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_jit_addr op2_def_addr,
uint32_t res_info,
zend_jit_addr res_addr,
zend_jit_addr ref_addr,
int may_throw)
{
ZEND_ASSERT(opline->op1_type == IS_CV);
if (op2_addr != op2_def_addr) {
if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
return 0;
}
if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
op2_addr = op2_def_addr;
}
}
if (Z_MODE(op1_addr) != IS_REG
&& Z_MODE(op1_use_addr) == IS_REG
&& !Z_LOAD(op1_use_addr)
&& !Z_STORE(op1_use_addr)) {
/* Force type update */
op1_info |= MAY_BE_UNDEF;
}
if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info,
opline->op2_type, op2_addr, op2_info, res_addr, ref_addr, may_throw)) {
return 0;
}
if (Z_MODE(op1_addr) == IS_REG) {
if (Z_STORE(op1_addr)) {
if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
return 0;
}
} else if ((op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& Z_MODE(op1_use_addr) == IS_MEM_ZVAL
&& Z_REG(op1_use_addr) == ZREG_FP
&& EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) {
/* We have to update type of CV because it may be captured by exception backtrace or released on RETURN */
if ((op1_def_info & MAY_BE_ANY) == MAY_BE_LONG) {
jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG);
if (JIT_G(current_frame)) {
SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_LONG, 1);
}
} else if ((op1_def_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE);
if (JIT_G(current_frame)) {
SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_DOUBLE, 1);
}
} else {
ZEND_UNREACHABLE();
}
}
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
static ir_op zend_jit_cmp_op(const zend_op *opline)
{
ir_op op;
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
op = IR_EQ;
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
op = IR_NE;
break;
case ZEND_IS_SMALLER:
op = IR_LT;
break;
case ZEND_IS_SMALLER_OR_EQUAL:
op = IR_LE;
break;
default:
ZEND_UNREACHABLE();
}
return op;
}
static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit,
const zend_op *opline,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
ir_ref ref;
bool result;
if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
if (!smart_branch_opcode ||
smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE);
}
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2);
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2);
} else {
ZEND_UNREACHABLE();
}
}
if (opline->opcode != ZEND_IS_IDENTICAL
&& opline->opcode != ZEND_IS_NOT_IDENTICAL
&& opline->opcode != ZEND_CASE_STRICT) {
return ir_END();
} else {
return IR_NULL; /* success */
}
}
ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
if (opline->opcode != ZEND_IS_IDENTICAL
&& opline->opcode != ZEND_IS_NOT_IDENTICAL
&& opline->opcode != ZEND_CASE_STRICT) {
return ir_END();
} else {
return IR_NULL; /* success */
}
}
static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_addr));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
return ir_END();
}
static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit, op2_addr)));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
return ir_END();
}
static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_addr));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
if (opline->opcode != ZEND_IS_IDENTICAL
&& opline->opcode != ZEND_IS_NOT_IDENTICAL
&& opline->opcode != ZEND_CASE_STRICT) {
return ir_END();
} else {
return IR_NULL; /* success */
}
}
static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
ref = ir_CMP_OP(zend_jit_cmp_op(opline), ref, ir_CONST_I32(0));
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
return jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
return ir_END();
}
static int zend_jit_cmp(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
ir_ref ref = IR_UNUSED;
ir_ref if_op1_long = IR_UNUSED;
ir_ref if_op1_double = IR_UNUSED;
ir_ref if_op2_double = IR_UNUSED;
ir_ref if_op1_long_op2_long = IR_UNUSED;
ir_ref if_op1_long_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_double = IR_UNUSED;
ir_ref if_op1_double_op2_long = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
bool has_slow =
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
ir_refs *end_inputs;
ir_refs_init(end_inputs, 8);
if (Z_MODE(op1_addr) == IS_REG) {
if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op1_addr);
}
} else if (Z_MODE(op2_addr) == IS_REG) {
if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) {
/* Force load */
zend_jit_use_reg(jit, op2_addr);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_op1_long);
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_long);
if (op2_info & MAY_BE_DOUBLE) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else {
ir_END_list(slow_inputs);
}
ir_IF_TRUE(if_op1_long_op2_long);
}
ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_long) {
ir_IF_FALSE_cold(if_op1_long);
}
if (op1_info & MAY_BE_DOUBLE) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if (if_op1_long) {
ir_END_list(slow_inputs);
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op1_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double);
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_double_op2_long);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_double_op2_long);
}
ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE);
ir_IF_FALSE_cold(if_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op2_double);
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE);
ir_IF_TRUE(if_op1_double_op2_double);
}
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
if (if_op1_double_op2_double) {
ir_IF_FALSE_cold(if_op1_double_op2_double);
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_op1_long_op2_double);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_op1_long_op2_double);
}
ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
} else if (if_op1_double_op2_double) {
ir_END_list(slow_inputs);
}
}
if (has_slow ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
ir_ref op1, op2, ref;
if (slow_inputs) {
ir_MERGE_list(slow_inputs);
}
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
op1 = jit_ZVAL_ADDR(jit, op1_addr);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0);
}
op2 = jit_ZVAL_ADDR(jit, op2_addr);
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0);
}
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_compare), op1, op2);
if (opline->opcode != ZEND_CASE) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
ir_refs_add(end_inputs, ref);
}
if (end_inputs->count) {
uint32_t n = end_inputs->count;
if (smart_branch_opcode && !exit_addr) {
zend_basic_block *bb;
ir_ref ref;
uint32_t label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label2 : target_label;
uint32_t label2 = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label : target_label2;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
if (UNEXPECTED(bb->successors[0] == bb->successors[1])) {
ir_ref merge_inputs = IR_UNUSED;
while (n) {
n--;
ir_IF_TRUE(end_inputs->refs[n]);
ir_END_list(merge_inputs);
ir_IF_FALSE(end_inputs->refs[n]);
ir_END_list(merge_inputs);
}
ir_MERGE_list(merge_inputs);
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
} else if (n == 1) {
ref = end_inputs->refs[0];
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
} else {
ir_ref true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
while (n) {
n--;
ir_IF_TRUE(end_inputs->refs[n]);
ir_END_list(true_inputs);
ir_IF_FALSE(end_inputs->refs[n]);
ir_END_list(false_inputs);
}
ir_MERGE_list(true_inputs);
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
ir_MERGE_list(false_inputs);
_zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END());
}
jit->b = -1;
} else {
ir_MERGE_N(n, end_inputs->refs);
}
} else if (smart_branch_opcode && !exit_addr) {
/* dead code */
_zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
jit->b = -1;
}
return 1;
}
static int zend_jit_identical(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
bool always_false = 0, always_true = 0;
ir_ref ref = IR_UNUSED;
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr);
op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0);
op1_info |= MAY_BE_NULL;
op1_addr = ZEND_ADDR_REF_ZVAL(op1);
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr);
op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0);
op2_info |= MAY_BE_NULL;
op2_addr = ZEND_ADDR_REF_ZVAL(op2);
}
if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
always_false = 1;
} else if (has_concrete_type(op1_info)
&& has_concrete_type(op2_info)
&& concrete_type(op1_info) == concrete_type(op2_info)
&& concrete_type(op1_info) <= IS_TRUE) {
always_true = 1;
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
always_true = 1;
} else {
always_false = 1;
}
}
if (always_true) {
if (opline->opcode != ZEND_CASE_STRICT) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!smart_branch_opcode
|| smart_branch_opcode == ZEND_JMPZ_EX
|| smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
uint32_t label;
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label : target_label2;
} else {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label2 : target_label;
}
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
jit->b = -1;
}
return 1;
} else if (always_false) {
if (opline->opcode != ZEND_CASE_STRICT) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!smart_branch_opcode
|| smart_branch_opcode == ZEND_JMPZ_EX
|| smart_branch_opcode == ZEND_JMPNZ_EX) {
jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
uint32_t label;
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label2 : target_label;
} else {
label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ?
target_label : target_label2;
}
_zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END());
jit->b = -1;
}
return 1;
}
if ((opline->op1_type & (IS_CV|IS_VAR)) && (op1_info & MAY_BE_REF)) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if ((opline->op2_type & (IS_CV|IS_VAR)) && (op2_info & MAY_BE_REF)) {
ref = jit_ZVAL_ADDR(jit, op2_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op2_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison);
if (!ref) {
return 0;
}
} else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr);
if (!ref) {
return 0;
}
} else {
if (opline->op1_type != IS_CONST) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
}
if (opline->op2_type != IS_CONST) {
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
}
}
if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op1_addr);
ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val)));
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op2_addr);
ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val)));
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) {
return 0;
}
op2_addr = real_addr;
}
if (may_throw) {
jit_SET_EX_OPLINE(jit, opline);
}
ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_is_identical),
jit_ZVAL_ADDR(jit, op1_addr),
jit_ZVAL_ADDR(jit, op2_addr));
}
if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) {
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
}
if (opline->opcode != ZEND_CASE_STRICT) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
if (opline->opcode == ZEND_IS_NOT_IDENTICAL) {
/* swap labels */
uint32_t tmp = target_label;
target_label = target_label2;
target_label2 = tmp;
}
ref = jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label);
}
}
if (smart_branch_opcode && !exit_addr) {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
if (bb->successors_count == 2 && bb->successors[0] == bb->successors[1]) {
ir_IF_TRUE(ref);
ir_MERGE_WITH_EMPTY_FALSE(ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
} else {
ZEND_ASSERT(bb->successors_count == 2);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
}
jit->b = -1;
}
return 1;
}
static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr)
{
uint32_t true_label = -1;
uint32_t false_label = -1;
bool set_bool = 0;
bool set_bool_not = 0;
bool always_true = 0, always_false = 0;
ir_ref ref, end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
ir_type type = IR_UNUSED;
if (branch_opcode == ZEND_BOOL) {
set_bool = 1;
} else if (branch_opcode == ZEND_BOOL_NOT) {
set_bool = 1;
set_bool_not = 1;
} else if (branch_opcode == ZEND_JMPZ) {
true_label = target_label2;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
false_label = target_label2;
} else if (branch_opcode == ZEND_JMPZ_EX) {
set_bool = 1;
true_label = target_label2;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ_EX) {
set_bool = 1;
true_label = target_label;
false_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
if (zend_is_true(Z_ZV(op1_addr))) {
always_true = 1;
} else {
always_false = 1;
}
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
always_true = 1;
} else if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0);
}
always_false = 1;
}
}
if (always_true) {
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
if (true_label != (uint32_t)-1) {
ZEND_ASSERT(exit_addr == NULL);
_zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END());
jit->b = -1;
}
return 1;
} else if (always_false) {
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
if (false_label != (uint32_t)-1) {
ZEND_ASSERT(exit_addr == NULL);
_zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END());
jit->b = -1;
}
return 1;
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
type = jit_Z_TYPE(jit, op1_addr);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) {
ir_ref if_type = ir_IF(ir_LT(type, ir_CONST_U8(IS_TRUE)));
ir_IF_TRUE_cold(if_type);
if (op1_info & MAY_BE_UNDEF) {
zend_jit_type_check_undef(jit,
type,
opline->op1.var,
opline, 1, 0, 1);
}
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
ir_END_list(end_inputs);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (false_label != (uint32_t)-1) {
ir_END_list(false_inputs);
} else {
ir_END_list(end_inputs);
}
ir_IF_FALSE(if_type);
}
if (op1_info & MAY_BE_TRUE) {
ir_ref if_type = IR_UNUSED;
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if_type = ir_IF(ir_EQ(type, ir_CONST_U8(IS_TRUE)));
ir_IF_TRUE(if_type);
}
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_END_list(end_inputs);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else if (true_label != (uint32_t)-1) {
ir_END_list(true_inputs);
} else {
ir_END_list(end_inputs);
}
if (if_type) {
ir_IF_FALSE(if_type);
}
}
}
if (op1_info & MAY_BE_LONG) {
ir_ref if_long = IR_UNUSED;
ir_ref ref;
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
if (!type) {
type = jit_Z_TYPE(jit, op1_addr);
}
if_long = ir_IF(ir_EQ(type, ir_CONST_U8(IS_LONG)));
ir_IF_TRUE(if_long);
}
ref = jit_Z_LVAL(jit, op1_addr);
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
ref = ir_NE(ref, ir_CONST_LONG(0));
if (set_bool_not) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
ir_END_list(end_inputs);
} else if (exit_addr) {
if (set_bool) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, ir_CONST_LONG(0))), ir_CONST_U32(IS_FALSE)));
}
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
ir_END_list(end_inputs);
} else {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
ir_END_list(false_inputs);
}
if (if_long) {
ir_IF_FALSE(if_long);
}
}
if (op1_info & MAY_BE_DOUBLE) {
ir_ref if_double = IR_UNUSED;
ir_ref ref;
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (!type) {
type = jit_Z_TYPE(jit, op1_addr);
}
if_double = ir_IF(ir_EQ(type, ir_CONST_U8(IS_DOUBLE)));
ir_IF_TRUE(if_double);
}
ref = ir_NE(jit_Z_DVAL(jit, op1_addr), ir_CONST_DOUBLE(0.0));
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
if (set_bool_not) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
ir_END_list(end_inputs);
} else if (exit_addr) {
if (set_bool) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
ir_END_list(end_inputs);
} else {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
ir_END_list(false_inputs);
}
if (if_double) {
ir_IF_FALSE(if_double);
}
}
if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) {
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr));
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) {
if (set_bool_not) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref)));
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (exit_addr) {
if (set_bool) {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
}
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
if (end_inputs) {
ir_END_list(end_inputs);
}
} else {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE);
}
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
if (set_bool) {
jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE);
}
ir_END_list(false_inputs);
}
}
if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
}
return 1;
}
static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t defined_label = (uint32_t)-1;
uint32_t undefined_label = (uint32_t)-1;
zval *zv = RT_CONSTANT(opline, opline->op1);
zend_jit_addr res_addr = 0;
ir_ref ref, ref2, if_set, if_zero, if_set2;
ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
defined_label = target_label2;
undefined_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
defined_label = target_label;
undefined_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
} else {
res_addr = RES_ADDR();
}
// if (CACHED_PTR(opline->extended_value)) {
ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
if_set = ir_IF(ref);
ir_IF_FALSE_cold(if_set);
if_zero = ir_END();
ir_IF_TRUE(if_set);
if_set2 = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
ir_IF_FALSE(if_set2);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
ir_IF_TRUE_cold(if_set2);
ref2 = jit_EG(zend_constants);
ref = ir_SHR_A(ref, ir_CONST_ADDR(1));
if (sizeof(void*) == 8) {
ref = ir_TRUNC_U32(ref);
}
ref2 = ir_EQ(ref, ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref2), offsetof(HashTable, nNumOfElements))));
ref2 = ir_IF(ref2);
ir_IF_TRUE(ref2);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
ir_IF_FALSE(ref2);
ir_MERGE_2(if_zero, ir_END());
jit_SET_EX_OPLINE(jit, opline);
ref2 = ir_NE(ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_check_constant), ir_CONST_ADDR(zv)), IR_NULL);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
ir_GUARD(ref2, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref2, ir_CONST_ADDR(exit_addr));
}
ir_END_list(end_inputs);
} else if (smart_branch_opcode) {
ref2 = ir_IF(ref2);
ir_IF_TRUE(ref2);
ir_END_list(true_inputs);
ir_IF_FALSE(ref2);
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref2), ir_CONST_U32(IS_FALSE)));
ir_END_list(end_inputs);
}
if (!smart_branch_opcode || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs);
}
return 1;
}
static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg)
{
zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr));
ir_IF_FALSE_cold(if_def);
if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
if (!zend_jit_save_call_chain(jit, -1)) {
return 0;
}
}
if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
&& (opline-1)->opcode != ZEND_FETCH_LIST_R
&& ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
zend_jit_zval_try_addref(jit, val_addr);
}
jit_LOAD_IP_ADDR(jit, opline - 1);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
ir_IF_TRUE(if_def);
return 1;
}
static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg));
// JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1);
return 1;
}
static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr val_addr,
uint8_t type,
bool deref,
uint32_t flags,
bool op1_avoid_refcounting)
{
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
const void *res_exit_addr = NULL;
ir_ref end1 = IR_UNUSED, ref1 = IR_UNUSED;
ir_ref ref = jit_ZVAL_ADDR(jit, val_addr);
uint32_t old_op1_info = 0;
uint32_t old_info;
ir_ref old_ref;
if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
if (op1_avoid_refcounting
|| ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS))) {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
}
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
if (deref) {
ir_ref if_type;
if (type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
if_type = ir_IF(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)));
} else {
if_type = jit_if_Z_TYPE(jit, val_addr, type);
}
ir_IF_TRUE(if_type);
end1 = ir_END();
ref1 = ref;
ir_IF_FALSE_cold(if_type);
SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!res_exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr);
ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val));
val_addr = ZEND_ADDR_REF_ZVAL(ref);
}
SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!res_exit_addr) {
return 0;
}
if (!deref && type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) {
ir_GUARD(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)), ir_CONST_ADDR(res_exit_addr));
} else {
jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr);
}
if (deref) {
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref, ref1);
}
val_addr = ZEND_ADDR_REF_ZVAL(ref);
SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), old_ref);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
}
return val_addr;
}
static int zend_jit_fetch_constant(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
zend_jit_addr res_addr)
{
zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
uint32_t res_info = RES_INFO();
ir_ref ref, ref2, if_set, if_special, not_set_path, special_path, fast_path;
// JIT: c = CACHED_PTR(opline->extended_value);
ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value));
// JIT: if (c != NULL)
if_set = ir_IF(ref);
if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
// JIT: if (!IS_SPECIAL_CACHE_VAL(c))
ir_IF_FALSE_cold(if_set);
not_set_path = ir_END();
ir_IF_TRUE(if_set);
if_special = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL)));
ir_IF_TRUE_cold(if_special);
special_path = ir_END();
ir_IF_FALSE(if_special);
fast_path = ir_END();
ir_MERGE_2(not_set_path, special_path);
} else {
ir_IF_TRUE(if_set);
fast_path = ir_END();
ir_IF_FALSE_cold(if_set);
}
// JIT: zend_jit_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num);
jit_SET_EX_OPLINE(jit, opline);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_constant),
ir_CONST_ADDR(zv),
ir_CONST_U32(opline->op1.num));
ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH(fast_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
uint8_t type = concrete_type(res_info);
zend_jit_addr const_addr = ZEND_ADDR_REF_ZVAL(ref);
const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0);
if (!const_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
// JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1);
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
} else {
ir_ref const_addr = ZEND_ADDR_REF_ZVAL(ref);
// JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1);
}
return 1;
}
static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t mask;
zend_jit_addr op1_addr = OP1_ADDR();
zend_jit_addr res_addr = 0;
uint32_t true_label = -1, false_label = -1;
ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
// TODO: support for is_resource() ???
ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
true_label = target_label2;
false_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
false_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
} else {
res_addr = RES_ADDR();
}
if (op1_info & MAY_BE_UNDEF) {
ir_ref if_def = IR_UNUSED;
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
}
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
zend_jit_check_exception_undef_result(jit, opline);
if (opline->extended_value & MAY_BE_NULL) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
if (if_def) {
ir_END_list(end_inputs);
}
}
}
if (if_def) {
ir_IF_TRUE(if_def);
op1_info |= MAY_BE_NULL;
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
mask = opline->extended_value;
if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
} else {
ir_ref ref;
bool invert = 0;
uint8_t type;
switch (mask) {
case MAY_BE_NULL: type = IS_NULL; break;
case MAY_BE_FALSE: type = IS_FALSE; break;
case MAY_BE_TRUE: type = IS_TRUE; break;
case MAY_BE_LONG: type = IS_LONG; break;
case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
case MAY_BE_STRING: type = IS_STRING; break;
case MAY_BE_ARRAY: type = IS_ARRAY; break;
case MAY_BE_OBJECT: type = IS_OBJECT; break;
case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break;
case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break;
case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break;
case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break;
case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break;
case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
default:
type = 0;
}
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (type == 0) {
ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask));
if (!smart_branch_opcode) {
ref = ir_NE(ref, ir_CONST_U32(0));
}
} else if (invert) {
ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
} else {
ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type));
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
if (end_inputs) {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
ir_END_list(end_inputs);
}
}
}
if (!smart_branch_opcode || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
} else if (exit_addr && !jit->ctx.control) {
ir_BEGIN(IR_UNUSED); /* unreachable block */
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
}
return 1;
}
static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_jit_addr res_addr = RES_ADDR();
uint32_t true_label = -1, false_label = -1;
ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED;
// TODO: support for empty() ???
ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
true_label = target_label2;
false_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
false_label = target_label2;
} else {
ZEND_UNREACHABLE();
}
} else {
res_addr = RES_ADDR();
}
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
} else if (smart_branch_opcode) {
ir_END_list(true_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
} else if (smart_branch_opcode) {
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
} else {
ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL));
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
ir_ref if_val = ir_IF(ref);
ir_IF_TRUE(if_val);
ir_END_list(true_inputs);
ir_IF_FALSE(if_val);
ir_END_list(false_inputs);
} else {
jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr),
ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE)));
ir_END_list(end_inputs);
}
}
if (!smart_branch_opcode || exit_addr) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
} else {
_zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs);
}
return 1;
}
/* copy of hidden zend_closure */
typedef struct _zend_closure {
zend_object std;
zend_function func;
zval this_ptr;
zend_class_entry *called_scope;
zif_handler orig_internal_handler;
} zend_closure;
static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
// JIT: if (EG(vm_stack_end) - EG(vm_stack_top) < used_stack)
ir_GUARD(
ir_UGE(
ir_SUB_A(ir_LOAD_A(jit_EG(vm_stack_end)), ir_LOAD_A(jit_EG(vm_stack_top))),
ir_CONST_ADDR(used_stack)),
ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_free_trampoline(zend_jit_ctx *jit, int8_t func_reg)
{
// JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
ir_ref func = ir_RLOAD_A(func_reg);
ir_ref if_trampoline = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE)));
ir_IF_TRUE(if_trampoline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_trampoline_helper), func);
ir_MERGE_WITH_EMPTY_FALSE(if_trampoline);
return 1;
}
static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack, ir_ref func_ref, ir_ref this_ref)
{
uint32_t used_stack;
ir_ref used_stack_ref = IR_UNUSED;
bool stack_check = 1;
ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED;
ZEND_ASSERT(func_ref != IR_NULL);
if (func) {
used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
if ((int)used_stack <= checked_stack) {
stack_check = 0;
}
used_stack_ref = ir_CONST_ADDR(used_stack);
} else {
ir_ref num_args_ref;
ir_ref if_internal_func = IR_UNUSED;
used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval);
used_stack_ref = ir_CONST_ADDR(used_stack);
if (!is_closure) {
used_stack_ref = ir_HARD_COPY_A(used_stack_ref); /* load constant once */
// JIT: if (EXPECTED(ZEND_USER_CODE(func->type))) {
ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(1)));
ir_IF_FALSE(if_internal_func);
}
// JIT: used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
num_args_ref = ir_CONST_U32(opline->extended_value);
if (!is_closure) {
ref = ir_SUB_U32(
ir_SUB_U32(
ir_MIN_U32(
num_args_ref,
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.num_args)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.last_var)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.T))));
} else {
ref = ir_SUB_U32(
ir_SUB_U32(
ir_MIN_U32(
num_args_ref,
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.num_args)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.last_var)))),
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.T))));
}
ref = ir_MUL_U32(ref, ir_CONST_U32(sizeof(zval)));
if (sizeof(void*) == 8) {
ref = ir_SEXT_A(ref);
}
ref = ir_SUB_A(used_stack_ref, ref);
if (is_closure) {
used_stack_ref = ref;
} else {
ir_MERGE_WITH_EMPTY_TRUE(if_internal_func);
used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref);
}
}
zend_jit_start_reuse_ip(jit);
// JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top)));
if (stack_check) {
// JIT: Check Stack Overflow
ref = ir_UGE(
ir_SUB_A(
ir_LOAD_A(jit_EG(vm_stack_end)),
jit_IP(jit)),
used_stack_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL);
int32_t exit_point = zend_jit_trace_get_exit_point(opline,
may_be_trampoline ?
(ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (may_be_trampoline) {
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
}
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
if_enough_stack = ir_IF(ref);
ir_IF_FALSE_cold(if_enough_stack);
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref);
} else {
if (!is_closure) {
ref = func_ref;
} else {
ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func));
}
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper),
used_stack_ref, ref);
}
jit_STORE_IP(jit, ref);
cold_path = ir_END();
ir_IF_TRUE(if_enough_stack);
}
}
ref = jit_EG(vm_stack_top);
rx = jit_IP(jit);
#if !OPTIMIZE_FOR_SIZE
/* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack);
* This vesions is longer but faster
* mov EG(vm_stack_top), %CALL
* lea size(%call), %tmp
* mov %tmp, EG(vm_stack_top)
*/
top = rx;
#else
/* JIT: EG(vm_stack_top) += used_stack;
* Use ir_emit() because ir_LOAD() makes load forwarding and doesn't allow load/store fusion
* mov EG(vm_stack_top), %CALL
* add $size, EG(vm_stack_top)
*/
top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref);
#endif
ir_STORE(ref, ir_ADD_A(top, used_stack_ref));
// JIT: zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
// JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
ir_STORE(jit_CALL(rx, This.u1.type_info), ir_CONST_U32(IS_UNDEF | ZEND_CALL_NESTED_FUNCTION));
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
if (cold_path) {
ir_MERGE_WITH(cold_path);
rx = jit_IP(jit);
}
// JIT: call->func = func;
ir_STORE(jit_CALL(rx, func), func_ref);
} else {
if (!is_closure) {
// JIT: call->func = func;
ir_STORE(jit_CALL(rx, func), func_ref);
} else {
// JIT: call->func = &closure->func;
ir_STORE(jit_CALL(rx, func), ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
}
if (cold_path) {
ir_MERGE_WITH(cold_path);
rx = jit_IP(jit);
}
}
if (opline->opcode == ZEND_INIT_METHOD_CALL) {
// JIT: Z_PTR(call->This) = obj;
ZEND_ASSERT(this_ref != IR_NULL);
ir_STORE(jit_CALL(rx, This.value.ptr), this_ref);
if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
// JIT: call->call_info |= ZEND_CALL_HAS_THIS;
ref = jit_CALL(rx, This.u1.type_info);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS));
} else {
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_HAS_THIS)));
}
} else {
if (opline->op1_type == IS_CV) {
// JIT: GC_ADDREF(obj);
jit_GC_ADDREF(jit, this_ref);
}
// JIT: call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
ref = jit_CALL(rx, This.u1.type_info);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS));
} else {
ir_STORE(ref,
ir_OR_U32(ir_LOAD_U32(ref),
ir_CONST_U32(ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)));
}
}
} else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) {
// JIT: Z_CE(call->This) = called_scope;
ir_STORE(jit_CALL(rx, This), this_ref);
} else if (!is_closure) {
// JIT: Z_CE(call->This) = called_scope;
ir_STORE(jit_CALL(rx, This), IR_NULL);
} else {
ir_ref object_or_called_scope, call_info, call_info2, object, if_cond;
if (opline->op2_type == IS_CV) {
// JIT: GC_ADDREF(closure);
jit_GC_ADDREF(jit, func_ref);
}
// JIT: RX(object_or_called_scope) = closure->called_scope;
object_or_called_scope = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, called_scope)));
// JIT: call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
// (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
call_info = ir_OR_U32(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.common.fn_flags))),
ir_CONST_U32(ZEND_ACC_FAKE_CLOSURE)),
ir_CONST_U32(ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE));
// JIT: if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
if_cond = ir_IF(ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.u1.v.type))));
ir_IF_TRUE(if_cond);
// JIT: call_info |= ZEND_CALL_HAS_THIS;
call_info2 = ir_OR_U32(call_info, ir_CONST_U32(ZEND_CALL_HAS_THIS));
// JIT: object_or_called_scope = Z_OBJ(closure->this_ptr);
object = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.value.ptr)));
ir_MERGE_WITH_EMPTY_FALSE(if_cond);
call_info = ir_PHI_2(IR_U32, call_info2, call_info);
object_or_called_scope = ir_PHI_2(IR_ADDR, object, object_or_called_scope);
// JIT: ZEND_SET_CALL_INFO(call, 0, call_info);
ref = jit_CALL(rx, This.u1.type_info);
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), call_info));
// JIT: Z_PTR(call->This) = object_or_called_scope;
ir_STORE(jit_CALL(rx, This.value.ptr), object_or_called_scope);
// JIT: if (closure->func.op_array.run_time_cache__ptr)
if_cond = ir_IF(ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.run_time_cache__ptr))));
ir_IF_FALSE(if_cond);
// JIT: zend_jit_init_func_run_time_cache_helper(closure->func);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper),
ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)));
ir_MERGE_WITH_EMPTY_TRUE(if_cond);
}
// JIT: ZEND_CALL_NUM_ARGS(call) = num_args;
ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value));
return 1;
}
static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, const zend_op *to_opline)
{
int32_t exit_point;
const void *exit_addr;
ir_ref call;
if (func->type == ZEND_INTERNAL_FUNCTION) {
#ifdef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
return 0;
#endif
} else if (func->type == ZEND_USER_FUNCTION) {
if (!zend_accel_in_shm(func->op_array.opcodes)) {
/* op_array and op_array->opcodes are not persistent. We can't link. */
return 0;
}
} else {
ZEND_UNREACHABLE();
return 0;
}
exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
// call = EX(call);
call = ir_LOAD_A(jit_EX(call));
while (level > 0) {
// call = call->prev_execute_data
call = ir_LOAD_A(jit_CALL(call, prev_execute_data));
level--;
}
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
// JIT: if (call->func.op_array.opcodes != opcodes) goto exit_addr;
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(call, func)), offsetof(zend_op_array, opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
} else {
// JIT: if (call->func != func) goto exit_addr;
ir_GUARD(ir_EQ(ir_LOAD_A(jit_CALL(call, func)), ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
}
return 1;
}
static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
ir_ref func_ref = IR_UNUSED;
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (!func
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL) {
#ifdef _WIN32
/* ASLR */
if (trace->func->type != ZEND_INTERNAL_FUNCTION) {
func = (zend_function*)trace->func;
}
#else
func = (zend_function*)trace->func;
#endif
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_INTERNAL_FUNCTION) {
#endif
/* load constant address later */
func_ref = ir_CONST_ADDR(func);
} else if (func && op_array == &func->op_array) {
/* recursive call */
if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
|| zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) {
func_ref = ir_LOAD_A(jit_EX(func));
} else {
func_ref = ir_CONST_ADDR(func);
}
} else {
ir_ref if_func, cache_slot_ref, ref;
// JIT: if (CACHED_PTR(opline->result.num))
cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num);
func_ref = ir_LOAD_A(cache_slot_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& func
&& (func->common.fn_flags & ZEND_ACC_IMMUTABLE)
&& opline->opcode != ZEND_INIT_FCALL) {
/* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */
if_func = ir_IF(ir_EQ(func_ref, ir_CONST_ADDR(func)));
} else {
if_func = ir_IF(func_ref);
}
ir_IF_FALSE_cold(if_func);
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_USER_FUNCTION
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
ref = ir_HARD_COPY_A(ir_CONST_ADDR(func)); /* load constant once */
ir_STORE(cache_slot_ref, ref);
ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper), ref);
} else {
zval *zv = RT_CONSTANT(opline, opline->op2);
if (opline->opcode == ZEND_INIT_FCALL) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
ir_CONST_ADDR(Z_STR_P(zv)),
cache_slot_ref);
} else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper),
ir_CONST_ADDR(Z_STR_P(zv + 1)),
cache_slot_ref);
} else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_ns_func_helper),
ir_CONST_ADDR(zv),
cache_slot_ref);
} else {
ZEND_UNREACHABLE();
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline,
func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (!func || opline->opcode == ZEND_INIT_FCALL) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else if (func->type == ZEND_USER_FUNCTION
&& !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
const zend_op *opcodes = func->op_array.opcodes;
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_op_array, opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ir_EQ(ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
}
} else {
jit_SET_EX_OPLINE(jit, opline);
ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function));
}
}
ir_MERGE_WITH_EMPTY_TRUE(if_func);
func_ref = ir_PHI_2(IR_ADDR, ref, func_ref);
}
if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSED)) {
return 0;
}
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(jit, call_level)) {
return 0;
}
} else {
ZEND_ASSERT(call_level > 0);
jit->delayed_call_level = call_level;
delayed_call_chain = 1;
}
return 1;
}
static int zend_jit_init_method_call(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
zend_jit_trace_rec *trace,
int checked_stack,
int8_t func_reg,
int8_t this_reg,
bool polymorphic_side_trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
zval *function_name;
ir_ref if_static = IR_UNUSED, cold_path, this_ref = IR_NULL, func_ref = IR_NULL;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
function_name = RT_CONSTANT(opline, opline->op2);
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (polymorphic_side_trace) {
/* function is passed in r0 from parent_trace */
ZEND_ASSERT(func_reg >= 0 && this_reg >= 0);
func_ref = zend_jit_deopt_rload(jit, IR_ADDR, func_reg);
this_ref = zend_jit_deopt_rload(jit, IR_ADDR, this_reg);
} else {
ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2;
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
this_ref = jit_Z_PTR(jit, this_addr);
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
// JIT: ZVAL_DEREF(op1)
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_ref if_ref;
/* Hack: Convert reference to regular value to simplify JIT code */
ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr));
ir_MERGE_WITH_EMPTY_FALSE(if_ref);
}
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)),
ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_object);
jit_SET_EX_OPLINE(jit, opline);
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call_tmp),
jit_ZVAL_ADDR(jit, op1_addr));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call),
jit_ZVAL_ADDR(jit, op1_addr));
}
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_TRUE(if_object);
}
}
this_ref = jit_Z_PTR(jit, op1_addr);
}
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (func) {
// JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*)));
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
fast_path = ir_END();
} else {
// JIT: if (CACHED_PTR(opline->result.num) == obj->ce)) {
run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ref = ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num)),
ir_LOAD_A(ir_ADD_OFFSET(this_ref, offsetof(zend_object, ce))));
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
// JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num + sizeof(void*)));
fast_path = ir_END();
}
ir_IF_FALSE_cold(if_found);
jit_SET_EX_OPLINE(jit, opline);
if (!jit->ctx.fixed_call_stack_size) {
// JIT: alloca(sizeof(void*));
this_ref2 = ir_ALLOCA(ir_CONST_ADDR(0x10));
} else {
this_ref2 = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
}
ir_STORE(this_ref2, this_ref);
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_tmp_helper),
this_ref,
ir_CONST_ADDR(function_name),
this_ref2);
} else {
ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_helper),
this_ref,
ir_CONST_ADDR(function_name),
this_ref2);
}
this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP));
if (!jit->ctx.fixed_call_stack_size) {
// JIT: revert alloca
ir_AFREE(ir_CONST_ADDR(0x10));
}
ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH(fast_path);
func_ref = ir_PHI_2(IR_ADDR, ref2, ref);
this_ref = ir_PHI_2(IR_ADDR, this_ref2, this_ref);
}
if ((!func || zend_jit_may_be_modified(func, op_array))
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
#ifdef _WIN32
&& trace->func->type != ZEND_INTERNAL_FUNCTION
#endif
) {
int32_t exit_point;
const void *exit_addr;
exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
jit->trace->exit_info[exit_point].poly_this_ref = this_ref;
func = (zend_function*)trace->func;
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ir_EQ(func_ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
}
}
if (!func) {
// JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
if_static = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_STATIC)));
ir_IF_TRUE_cold(if_static);
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
ir_ref ret;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame_tmp),
this_ref,
func_ref,
ir_CONST_U32(opline->extended_value));
} else {
ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame),
this_ref,
func_ref,
ir_CONST_U32(opline->extended_value));
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
jit_STORE_IP(jit, ret);
}
if (!func) {
cold_path = ir_END();
ir_IF_FALSE(if_static);
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) {
return 0;
}
}
if (!func) {
ir_MERGE_WITH(cold_path);
}
zend_jit_start_reuse_ip(jit);
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(jit, call_level)) {
return 0;
}
} else {
ZEND_ASSERT(call_level > 0);
delayed_call_chain = 1;
jit->delayed_call_level = call_level;
}
if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_ip(jit, opline + 1)) {
return 0;
}
}
return 1;
}
static int zend_jit_init_static_method_call(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
zend_jit_trace_rec *trace,
int checked_stack)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_class_entry *ce;
zend_function *func = NULL;
ir_ref func_ref, func_ref2, scope_ref, scope_ref2, if_cached, cold_path, ref;
ir_ref if_static = IR_UNUSED;
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
ce = zend_get_known_class(op_array, opline, opline->op1_type, opline->op1);
if (!func && ce) {
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_string *method_name;
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
method_name = Z_STR_P(zv);
zv = zend_hash_find(&ce->function_table, method_name);
if (zv) {
zend_function *fn = Z_PTR_P(zv);
if (fn->common.scope == op_array->scope
|| (fn->common.fn_flags & ZEND_ACC_PUBLIC)
|| ((fn->common.fn_flags & ZEND_ACC_PROTECTED)
&& op_array->scope
&& instanceof_function_slow(op_array->scope, fn->common.scope))) {
func = fn;
}
}
}
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
// JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*));
func_ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*)));
// JIT: if (fbc)
if_cached = ir_IF(func_ref);
ir_IF_FALSE_cold(if_cached);
jit_SET_EX_OPLINE(jit, opline);
scope_ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_class_helper), jit_FP(jit));
ir_GUARD(scope_ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
func_ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_static_method_helper), jit_FP(jit), scope_ref2);
ir_GUARD(func_ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler));
cold_path = ir_END();
ir_IF_TRUE(if_cached);
if (ce && (ce->ce_flags & ZEND_ACC_IMMUTABLE) && (ce->ce_flags & ZEND_ACC_LINKED)) {
scope_ref = ir_CONST_ADDR(ce);
} else {
scope_ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num));
}
ir_MERGE_2(cold_path, ir_END());
func_ref = ir_PHI_2(IR_ADDR, func_ref2, func_ref);
scope_ref = ir_PHI_2(IR_ADDR, scope_ref2, scope_ref);
if ((!func || zend_jit_may_be_modified(func, op_array))
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
#ifdef _WIN32
&& trace->func->type != ZEND_INTERNAL_FUNCTION
#endif
) {
int32_t exit_point;
const void *exit_addr;
exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
// jit->trace->exit_info[exit_point].poly_func_ref = func_ref;
// jit->trace->exit_info[exit_point].poly_this_ref = scope_ref;
func = (zend_function*)trace->func;
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD(ir_EQ(func_ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr));
}
}
if (!func || !(func->common.fn_flags & ZEND_ACC_STATIC)) {
if (!func) {
// JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
if_static = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))),
ir_CONST_U32(ZEND_ACC_STATIC)));
ir_IF_FALSE_cold(if_static);
}
ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_this_method_call_frame),
scope_ref,
func_ref,
ir_CONST_U32(opline->extended_value));
ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
jit_STORE_IP(jit, ref);
if (!func) {
cold_path = ir_END();
ir_IF_TRUE(if_static);
}
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC)) {
if (opline->op1_type == IS_UNUSED
&& ((opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT ||
(opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF)) {
if (op_array->fn_flags & ZEND_ACC_STATIC) {
scope_ref = ir_LOAD_A(jit_EX(This.value.ref));
} else {
scope_ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(This.value.ref)), offsetof(zend_object, ce)));
}
}
if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, scope_ref)) {
return 0;
}
if (!func) {
ir_MERGE_2(cold_path, ir_END());
}
}
zend_jit_start_reuse_ip(jit);
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(jit, call_level)) {
return 0;
}
} else {
ZEND_ASSERT(call_level > 0);
jit->delayed_call_level = call_level;
delayed_call_chain = 1;
}
return 1;
}
static int zend_jit_init_closure_call(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
zend_jit_trace_rec *trace,
int checked_stack)
{
zend_function *func = NULL;
zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
ir_ref ref;
ref = jit_Z_PTR(jit, op2_addr);
if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
&& !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_object, ce))),
ir_CONST_ADDR(zend_ce_closure)),
ir_CONST_ADDR(exit_addr));
if (ssa->var_info && ssa_op->op2_use >= 0) {
ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
}
}
if (trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
&& trace->func->type == ZEND_USER_FUNCTION) {
const zend_op *opcodes;
int32_t exit_point;
const void *exit_addr;
func = (zend_function*)trace->func;
opcodes = func->op_array.opcodes;
exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(
ir_EQ(
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_closure, func.op_array.opcodes))),
ir_CONST_ADDR(opcodes)),
ir_CONST_ADDR(exit_addr));
}
if (jit->delayed_call_level) {
if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) {
return 0;
}
}
if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) {
return 0;
}
if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(jit, call_level)) {
return 0;
}
} else {
ZEND_ASSERT(call_level > 0);
delayed_call_chain = 1;
jit->delayed_call_level = call_level;
}
if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_ip(jit, opline + 1)) {
return 0;
}
}
return 1;
}
static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAL_EX) {
uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
/* Don't generate code that always throws exception */
return 0;
}
} else {
ir_ref cond = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask));
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_pass_by_ref;
if_pass_by_ref = ir_IF(cond);
ir_IF_TRUE_cold(if_pass_by_ref);
if (Z_MODE(op1_addr) == IS_REG) {
/* set type to avoid zval_ptr_dtor() on uninitialized value */
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF);
}
jit_SET_EX_OPLINE(jit, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref));
ir_IF_FALSE(if_pass_by_ref);
}
}
}
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
jit_ZVAL_COPY_CONST(jit,
arg_addr,
MAY_BE_ANY, MAY_BE_ANY,
zv, 1);
} else {
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
}
return 1;
}
static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
{
zend_jit_addr op1_addr, arg_addr, ref_addr;
ir_ref ref_path = IR_UNUSED;
op1_addr = OP1_ADDR();
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
if (opline->op1_type == IS_VAR) {
if (op1_info & MAY_BE_INDIRECT) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
// JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE(if_def);
// JIT: ZVAL_NULL(op1)
jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL);
ir_MERGE_WITH_EMPTY_TRUE(if_def);
}
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
}
} else {
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
ir_ref ref, ref2;
if (op1_info & MAY_BE_REF) {
ir_ref if_ref;
// JIT: if (Z_TYPE_P(op1) == IS_UNDEF)
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
// JIT: ref = Z_PTR_P(op1)
ref = jit_Z_PTR(jit, op1_addr);
// JIT: GC_ADDREF(ref)
jit_GC_ADDREF(jit, ref);
// JIT: ZVAL_REFERENCE(arg, ref)
jit_set_Z_PTR(jit, arg_addr, ref);
jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
ref_path = ir_END();
ir_IF_FALSE(if_ref);
}
// JIT: ZVAL_NEW_REF(arg, varptr);
// JIT: ref = emalloc(sizeof(zend_reference));
ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline);
// JIT: GC_REFCOUNT(ref) = 2
jit_set_GC_REFCOUNT(jit, ref, 2);
// JIT: GC_TYPE(ref) = GC_REFERENCE
ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, gc.u.type_info)), ir_CONST_U32(GC_REFERENCE));
ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr)), IR_NULL);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
// JIT: ZVAL_COPY_VALUE(&ref->val, op1)
jit_ZVAL_COPY(jit,
ref_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
// JIT: ZVAL_REFERENCE(arg, ref)
jit_set_Z_PTR(jit, op1_addr, ref);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
// JIT: ZVAL_REFERENCE(arg, ref)
jit_set_Z_PTR(jit, arg_addr, ref);
jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX);
}
if (ref_path) {
ir_MERGE_WITH(ref_path);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
return 1;
}
static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ir_ref end_inputs = IR_UNUSED;
ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
arg_num <= MAX_ARG_FLAG_NUM);
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAR_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
// JIT: if (RX->func->quick_arg_flags & mask)
ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask)));
ir_IF_TRUE_cold(if_send_by_ref);
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
return 0;
}
ir_END_list(end_inputs);
ir_IF_FALSE(if_send_by_ref);
}
} else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!(op1_info & MAY_BE_REF)) {
/* Don't generate code that always throws exception */
return 0;
} else {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
// JIT: if (Z_TYPE_P(op1) != IS_REFERENCE)
ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)),
ir_CONST_ADDR(exit_addr));
}
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
ir_ref func, if_send_by_ref, if_prefer_ref;
// JIT: if (RX->func->quick_arg_flags & mask)
func = ir_LOAD_A(jit_RX(func));
if_send_by_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask)));
ir_IF_TRUE_cold(if_send_by_ref);
mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
if (op1_info & MAY_BE_REF) {
ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
}
// JIT: if (RX->func->quick_arg_flags & mask)
if_prefer_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask)));
ir_IF_TRUE(if_prefer_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_prefer_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
jit_ZVAL_ADDR(jit, arg_addr));
zend_jit_check_exception(jit);
ir_END_list(end_inputs);
}
ir_IF_FALSE(if_send_by_ref);
}
} else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
// JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF)
ir_ref if_send_by_ref = ir_IF(ir_AND_U32(
ir_LOAD_U32(jit_RX(This.u1.type_info)),
ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
ir_IF_TRUE_cold(if_send_by_ref);
if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) {
return 0;
}
ir_END_list(end_inputs);
ir_IF_FALSE(if_send_by_ref);
}
}
if (op1_info & MAY_BE_UNDEF) {
ir_ref ref, if_def = IR_UNUSED;
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF);
ir_IF_FALSE_cold(if_def);
}
// JIT: zend_jit_undefined_op_helper(opline->op1.var)
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
ir_CONST_U32(opline->op1.var));
// JIT: ZVAL_NULL(arg)
jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL);
// JIT: check_exception
ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
ir_END_list(end_inputs);
ir_IF_TRUE(if_def);
} else {
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
}
if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
if (op1_info & MAY_BE_REF) {
// JIT: if (Z_TYPE_P(arg) == IS_REFERENCE)
ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE);
ir_IF_TRUE(if_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(IR_FALSE, ir_CONST_ADDR(exit_addr));
} else {
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference),
jit_ZVAL_ADDR(jit, arg_addr));
zend_jit_check_exception(jit);
}
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
ir_ref ref;
// JIT: ZVAL_DEREF(op1)
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
// JIT: ZVAL_COPY(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 1);
} else {
ir_ref if_ref, ref, ref2, refcount, if_not_zero, if_refcounted;
zend_jit_addr ref_addr;
// JIT: if (Z_TYPE_P(op1) == IS_REFERENCE)
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE_cold(if_ref);
// JIT: ref = Z_COUNTED_P(op1);
ref = jit_Z_PTR(jit, op1_addr);
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
// JIT: ZVAL_COPY_VALUE(arg, op1);
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
ref_addr, op1_info, 0);
// JIT: if (GC_DELREF(ref) != 0)
refcount = jit_GC_DELREF(jit, ref);
if_not_zero = ir_IF(refcount);
ir_IF_TRUE(if_not_zero);
// JIT: if (Z_REFCOUNTED_P(arg)
if_refcounted = jit_if_REFCOUNTED(jit, arg_addr);
ir_IF_TRUE(if_refcounted);
// JIT: Z_ADDREF_P(arg)
jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr));
ir_END_list(end_inputs);
ir_IF_FALSE(if_refcounted);
ir_END_list(end_inputs);
ir_IF_FALSE(if_not_zero);
// JIT: efree(ref)
jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
ir_END_list(end_inputs);
ir_IF_FALSE(if_ref);
// JIT: ZVAL_COPY_VALUE(arg, op1);
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, 0);
}
} else {
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr = op1_def_addr;
}
}
// JIT: ZVAL_COPY_VALUE(arg, op1)
jit_ZVAL_COPY(jit,
arg_addr,
MAY_BE_ANY,
op1_addr, op1_info, opline->op1_type == IS_CV);
}
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline)
{
uint32_t arg_num = opline->op2.num;
ir_ref ref;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
// JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
if (jit->reuse_ip) {
ref = jit_IP(jit);
} else {
ref = ir_LOAD_A(jit_EX(call));
}
ref = jit_CALL(ref, This.u1.type_info);
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
}
} else {
if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
// JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
if (jit->reuse_ip) {
ref = jit_IP(jit);
} else {
ref = ir_LOAD_A(jit_EX(call));
}
ref = jit_CALL(ref, This.u1.type_info);
ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
}
}
} else {
// JIT: if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
ir_ref rx, if_ref, cold_path;
if (!zend_jit_reuse_ip(jit)) {
return 0;
}
rx = jit_IP(jit);
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))),
ir_CONST_U32(mask));
if_ref = ir_IF(ref);
ir_IF_TRUE_cold(if_ref);
// JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
ref = jit_CALL(rx, This.u1.type_info);
ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF)));
cold_path = ir_END();
ir_IF_FALSE(if_ref);
// JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
ref = jit_CALL(rx, This.u1.type_info);
ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF)));
ir_MERGE_WITH(cold_path);
}
return 1;
}
static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline)
{
ir_ref call, if_may_have_undef, ret;
if (jit->reuse_ip) {
call = jit_IP(jit);
} else {
call = ir_LOAD_A(jit_EX(call));
}
if_may_have_undef = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)),
ir_CONST_U8(ZEND_CALL_MAY_HAVE_UNDEF >> 24)));
ir_IF_TRUE_cold(if_may_have_undef);
jit_SET_EX_OPLINE(jit, opline);
ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_handle_undef_args), call);
ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH_EMPTY_FALSE(if_may_have_undef);
return 1;
}
static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
const zend_function *func = NULL;
uint32_t i;
uint32_t call_num_args = 0;
bool unknown_num_args = 0;
const void *exit_addr = NULL;
const zend_op *prev_opline;
ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED;
prev_opline = opline - 1;
while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
prev_opline--;
}
if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
unknown_num_args = 1;
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_call_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& !JIT_G(current_frame)->call->func) {
call_info = NULL; func = NULL; /* megamorphic call from trait */
}
}
if (!func) {
/* resolve function at run time */
} else if (func->type == ZEND_USER_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
call_num_args = call_info->num_args;
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
call_num_args = call_info->num_args;
} else {
ZEND_UNREACHABLE();
}
if (trace && !func) {
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
ZEND_ASSERT(!trace->func || trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
#endif
} else if (trace->op == ZEND_JIT_TRACE_ENTER) {
ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
}
}
}
bool may_have_extra_named_params =
opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
(!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
if (!jit->reuse_ip) {
zend_jit_start_reuse_ip(jit);
// JIT: call = EX(call);
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call)));
}
rx = jit_IP(jit);
zend_jit_stop_reuse_ip(jit);
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
func_ref = ir_LOAD_A(jit_CALL(rx, func));
ir_GUARD_NOT(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_DEPRECATED)),
ir_CONST_ADDR(exit_addr));
}
}
}
if (!jit->delayed_call_level) {
// JIT: EX(call) = call->prev_execute_data;
ir_STORE(jit_EX(call),
(call_level == 1) ? IR_NULL : ir_LOAD_A(jit_CALL(rx, prev_execute_data)));
}
delayed_call_chain = 0;
jit->delayed_call_level = 0;
// JIT: call->prev_execute_data = execute_data;
ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit));
if (!func) {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
}
if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
if (!trace) {
ir_ref if_deprecated, ret;
if_deprecated = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_DEPRECATED)));
ir_IF_TRUE_cold(if_deprecated);
if (GCC_GLOBAL_REGS) {
ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
} else {
ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
}
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_MERGE_WITH_EMPTY_FALSE(if_deprecated);
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
ir_ref ret;
if (GCC_GLOBAL_REGS) {
ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper));
} else {
ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx);
}
ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
}
if (!func
&& opline->opcode != ZEND_DO_UCALL
&& opline->opcode != ZEND_DO_ICALL) {
ir_ref type_ref = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type)));
if_user = ir_IF(ir_EQ(type_ref, ir_CONST_U8(ZEND_USER_FUNCTION)));
ir_IF_TRUE(if_user);
}
if ((!func || func->type == ZEND_USER_FUNCTION)
&& opline->opcode != ZEND_DO_ICALL) {
bool recursive_call_through_jmp = 0;
// JIT: EX(call) = NULL;
ir_STORE(jit_CALL(rx, call), IR_NULL);
// JIT: EX(return_value) = RETURN_VALUE_USED(opline) ? EX_VAR(opline->result.var) : 0;
ir_STORE(jit_CALL(rx, return_value),
RETURN_VALUE_USED(opline) ?
jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) :
IR_NULL);
// JIT: EX_LOAD_RUN_TIME_CACHE(op_array);
if (!func || func->op_array.cache_size) {
ir_ref run_time_cache;
if (func && op_array == &func->op_array) {
/* recursive call */
run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
} else if (func
&& !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)
&& ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)),
(uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)));
} else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) ||
(JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) {
/* Closures always use direct pointers */
ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
} else {
ir_ref if_odd, run_time_cache2;
ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func));
run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr)));
if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1)));
ir_IF_TRUE(if_odd);
run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base))));
ir_MERGE_WITH_EMPTY_FALSE(if_odd);
run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache2, run_time_cache);
}
ir_STORE(jit_CALL(rx, run_time_cache), run_time_cache);
}
// JIT: EG(current_execute_data) = execute_data = call;
ir_STORE(jit_EG(current_execute_data), rx);
jit_STORE_FP(jit, rx);
// JIT: opline = op_array->opcodes;
if (func && !unknown_num_args) {
for (i = call_num_args; i < func->op_array.last_var; i++) {
uint32_t n = EX_NUM_TO_VAR(i);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, n);
jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF));
}
if (call_num_args <= func->op_array.num_args) {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
uint32_t num_args;
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
if (trace) {
num_args = 0;
} else if (call_info) {
num_args = skip_valid_arguments(op_array, ssa, call_info);
} else {
num_args = call_num_args;
}
} else {
num_args = call_num_args;
}
if (zend_accel_in_shm(func->op_array.opcodes)) {
jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args);
} else {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
ir_ref ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
if (num_args) {
ip = ir_ADD_OFFSET(ip, num_args * sizeof(zend_op));
}
jit_LOAD_IP(jit, ip);
}
if (!trace && op_array == &func->op_array && call_num_args >= op_array->required_num_args) {
/* recursive call */
recursive_call_through_jmp = 1;
}
}
} else {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ir_ref ip;
if (zend_accel_in_shm(func->op_array.opcodes)) {
ip = ir_CONST_ADDR(func->op_array.opcodes);
} else {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
}
jit_LOAD_IP(jit, ip);
}
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
}
}
} else {
ir_ref ip;
ir_ref merge_inputs = IR_UNUSED;
// JIT: opline = op_array->opcodes
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
ip = ir_CONST_ADDR(func->op_array.opcodes);
} else {
if (!func_ref) {
func_ref = ir_LOAD_A(jit_CALL(rx, func));
}
ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes)));
}
jit_LOAD_IP(jit, ip);
// JIT: num_args = EX_NUM_ARGS();
ir_ref num_args, first_extra_arg;
num_args = ir_LOAD_U32(jit_EX(This.u2.num_args));
if (func) {
first_extra_arg = ir_CONST_U32(func->op_array.num_args);
} else {
// JIT: first_extra_arg = op_array->num_args;
ZEND_ASSERT(func_ref);
first_extra_arg = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, num_args)));
}
// JIT: if (UNEXPECTED(num_args > first_extra_arg))
ir_ref if_extra_args = ir_IF(ir_GT(num_args, first_extra_arg));
ir_IF_TRUE_cold(if_extra_args);
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit));
}
ir_END_list(merge_inputs);
ir_IF_FALSE(if_extra_args);
if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
if (!func) {
// JIT: if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
ir_ref if_has_type_hints = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))),
ir_CONST_U32(ZEND_ACC_HAS_TYPE_HINTS)));
ir_IF_TRUE(if_has_type_hints);
ir_END_list(merge_inputs);
ir_IF_FALSE(if_has_type_hints);
}
// JIT: opline += num_args;
ir_ref ref = ir_MUL_U32(num_args, ir_CONST_U32(sizeof(zend_op)));
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(ref);
}
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref));
} else {
ir_ref addr = jit_EX(opline);
ir_STORE(addr, ir_ADD_A(ir_LOAD_A(addr), ref));
}
}
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
// JIT: if (EXPECTED((int)num_args < op_array->last_var)) {
ir_ref last_var;
if (func) {
last_var = ir_CONST_U32(func->op_array.last_var);
} else {
ZEND_ASSERT(func_ref);
last_var = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, last_var)));
}
ir_ref idx = ir_SUB_U32(last_var, num_args);
ir_ref if_need = ir_IF(ir_GT(idx, ir_CONST_U32(0)));
ir_IF_TRUE(if_need);
// JIT: zval *var = EX_VAR_NUM(num_args);
if (sizeof(void*) == 8) {
num_args = ir_ZEXT_A(num_args);
}
ir_ref var_ref = ir_ADD_OFFSET(
ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))),
(ZEND_CALL_FRAME_SLOT * sizeof(zval)) + offsetof(zval, u1.type_info));
ir_ref loop = ir_LOOP_BEGIN(ir_END());
var_ref = ir_PHI_2(IR_ADDR, var_ref, IR_UNUSED);
idx = ir_PHI_2(IR_U32, idx, IR_UNUSED);
ir_STORE(var_ref, ir_CONST_I32(IS_UNDEF));
ir_PHI_SET_OP(var_ref, 2, ir_ADD_OFFSET(var_ref, sizeof(zval)));
ir_ref idx2 = ir_SUB_U32(idx, ir_CONST_U32(1));
ir_PHI_SET_OP(idx, 2, idx2);
ir_ref if_not_zero = ir_IF(idx2);
ir_IF_TRUE(if_not_zero);
ir_MERGE_SET_OP(loop, 2, ir_LOOP_END());
ir_IF_FALSE(if_not_zero);
ir_MERGE_WITH_EMPTY_FALSE(if_need);
}
if (ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0)) {
ir_ref observer_handler;
ir_ref rx = jit_FP(jit);
struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref);
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
jit_SET_EX_OPLINE(jit, trace[1].opline);
} else if (GCC_GLOBAL_REGS) {
// EX(opline) = opline
ir_STORE(jit_EX(opline), jit_IP(jit));
}
jit_observer_fcall_begin(jit, rx, observer_handler);
if (trace) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
exit_addr = NULL;
}
zend_jit_check_timeout(jit, NULL /* we're inside the called function */, exit_addr);
jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
}
if (trace) {
if (!func && (opline->opcode != ZEND_DO_UCALL)) {
user_path = ir_END();
}
} else {
zend_basic_block *bb;
do {
if (recursive_call_through_jmp) {
ir_ref begin, end;
ir_insn *insn;
/* attempt to convert direct recursive call into loop */
begin = jit->bb_start_ref[call_num_args];
ZEND_ASSERT(begin != IR_UNUSED);
insn = &jit->ctx.ir_base[begin];
if (insn->op == IR_BEGIN) {
end = ir_LOOP_END();
insn = &jit->ctx.ir_base[begin];
insn->op = IR_LOOP_BEGIN;
insn->inputs_count = 2;
insn->op2 = end;
break;
} else if ((insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN)
&& insn->inputs_count == 2) {
end = ir_LOOP_END();
insn = &jit->ctx.ir_base[begin];
insn->op = IR_LOOP_BEGIN;
insn->inputs_count = 3;
insn->op3 = end;
break;
} else if (insn->op == IR_LOOP_BEGIN && insn->inputs_count == 3) {
ZEND_ASSERT(jit->ctx.ir_base[insn->op3].op == IR_LOOP_END);
jit->ctx.ir_base[insn->op3].op = IR_END;
ir_MERGE_2(insn->op3, ir_END());
end = ir_LOOP_END();
insn = &jit->ctx.ir_base[begin];
insn->op3 = end;
break;
}
}
/* fallback to indirect JMP or RETURN */
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(1));
}
} while (0);
bb = &jit->ssa->cfg.blocks[jit->b];
if (bb->successors_count > 0) {
int succ;
ir_ref ref;
ZEND_ASSERT(bb->successors_count == 1);
succ = bb->successors[0];
/* Add a fake control edge from UNREACHABLE/RETURN to the following ENTRY */
ref = jit->ctx.insns_count - 1;
ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE
|| jit->ctx.ir_base[ref].op == IR_RETURN
|| jit->ctx.ir_base[ref].op == IR_LOOP_END);
ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY);
ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start);
if (func || (opline->opcode == ZEND_DO_UCALL)) {
_zend_jit_add_predecessor_ref(jit, succ, jit->b, ref);
jit->b = -1;
} else {
user_path = ref;
}
}
}
}
if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
&& (opline->opcode != ZEND_DO_UCALL)) {
if (!func && (opline->opcode != ZEND_DO_ICALL)) {
ir_IF_FALSE(if_user);
}
// JIT: EG(current_execute_data) = execute_data;
ir_STORE(jit_EG(current_execute_data), rx);
bool may_have_observer = ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0);
if (may_have_observer) {
ir_ref observer_handler;
struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func)));
jit_observer_fcall_begin(jit, rx, observer_handler);
jit_observer_fcall_is_unobserved_end(jit, &unobserved_data);
}
// JIT: ZVAL_NULL(EX_VAR(opline->result.var));
ir_ref res_addr = IR_UNUSED, func_ptr;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
} else {
/* CPU stack allocated temporary zval */
ir_ref ptr;
if (!jit->ctx.fixed_call_stack_size) {
// JIT: alloca(sizeof(void*));
ptr = ir_ALLOCA(ir_CONST_ADDR(sizeof(zval)));
} else {
ptr = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP));
}
res_addr = ZEND_ADDR_REF_ZVAL(ptr);
}
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
zend_jit_reset_last_valid_opline(jit);
// JIT: (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret);
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
if (zend_execute_internal) {
ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_execute_internal), rx, res_ref);
} else {
if (func) {
func_ptr = ir_CONST_FC_FUNC(func->internal_function.handler);
} else {
func_ptr = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler)));
#if defined(IR_TARGET_X86)
func_ptr = ir_CAST_FC_FUNC(func_ptr);
#endif
}
ir_CALL_2(IR_VOID, func_ptr, rx, res_ref);
}
if (may_have_observer) {
jit_observer_fcall_end(jit, rx, res_ref);
}
/* When zend_interrupt_function is set, it gets called while
* the frame is still on top. This is less efficient than
* doing it later once it's popped off. There is code further
* down that handles when there isn't an interrupt function.
*/
if (zend_interrupt_function) {
// JIT: if (EG(vm_interrupt)) zend_fcall_interrupt(execute_data);
ir_ref if_interrupt = ir_IF(ir_LOAD_U8(jit_EG(vm_interrupt)));
ir_IF_TRUE_cold(if_interrupt);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_fcall_interrupt), rx);
ir_MERGE_WITH_EMPTY_FALSE(if_interrupt);
}
// JIT: EG(current_execute_data) = execute_data;
ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
// JIT: zend_vm_stack_free_args(call);
if (func && !unknown_num_args) {
for (i = 0; i < call_num_args; i++ ) {
if (zend_jit_needs_arg_dtor(func, i, call_info)) {
uint32_t offset = EX_NUM_TO_VAR(i);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset);
jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline);
}
}
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_vm_stack_free_args_helper), rx);
}
if (may_have_extra_named_params) {
// JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS))
ir_ref if_has_named = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 3)),
ir_CONST_U8(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24)));
ir_IF_TRUE_cold(if_has_named);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_extra_named_params),
ir_LOAD_A(jit_CALL(rx, extra_named_params)));
ir_MERGE_WITH_EMPTY_FALSE(if_has_named);
}
if (opline->opcode == ZEND_DO_FCALL) {
// TODO: optimize ???
// JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
ir_ref if_release_this = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
ir_CONST_U8(ZEND_CALL_RELEASE_THIS >> 16)));
ir_IF_TRUE_cold(if_release_this);
// JIT: OBJ_RELEASE(Z_OBJ(RX->This));
jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj)));
ir_MERGE_WITH_EMPTY_FALSE(if_release_this);
}
ir_ref allocated_path = IR_UNUSED;
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!JIT_G(current_frame)->call ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
prev_opline->opcode == ZEND_SEND_UNPACK ||
prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
// JIT: zend_vm_stack_free_call_frame(call);
// JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_ALLOCATED))
ir_ref if_allocated = ir_IF(ir_AND_U8(
ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)),
ir_CONST_U8(ZEND_CALL_ALLOCATED >> 16)));
ir_IF_TRUE_cold(if_allocated);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_call_frame), rx);
allocated_path = ir_END();
ir_IF_FALSE(if_allocated);
}
ir_STORE(jit_EG(vm_stack_top), rx);
if (allocated_path) {
ir_MERGE_WITH(allocated_path);
}
if (!RETURN_VALUE_USED(opline)) {
zend_class_entry *ce;
bool ce_is_instanceof;
uint32_t func_info = call_info ?
zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
(MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
/* If an exception is thrown, the return_value may stay at the
* original value of null. */
func_info |= MAY_BE_NULL;
if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
ir_ref sp = ir_RLOAD_A(IR_REG_SP);
res_addr = ZEND_ADDR_REF_ZVAL(sp);
jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline);
}
if (!jit->ctx.fixed_call_stack_size) {
// JIT: revert alloca
ir_AFREE(ir_CONST_ADDR(sizeof(zval)));
}
}
// JIT: if (UNEXPECTED(EG(exception) != NULL)) {
ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)),
jit_STUB_ADDR(jit, jit_stub_icall_throw));
/* If there isn't a zend_interrupt_function, the timeout is
* handled here because it's more efficient.
*/
if (!zend_interrupt_function) {
// TODO: Can we avoid checking for interrupts after each call ???
if (trace && jit->last_valid_opline != opline) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
exit_addr = NULL;
}
zend_jit_check_timeout(jit, opline + 1, exit_addr);
}
if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
jit_LOAD_IP_ADDR(jit, opline + 1);
} else if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) {
jit_LOAD_IP_ADDR(jit, opline + 1);
}
}
if (user_path) {
ir_MERGE_WITH(user_path);
}
return 1;
}
static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, int next_block)
{
ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block);
ir_IF_FALSE(if_skip_constructor);
if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) {
if (!zend_jit_tail_handler(jit, opline)) {
return 0;
}
} else {
if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) {
return 0;
}
}
/* override predecessors of the next block */
ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1);
if (!jit->ctx.control) {
ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]);
ir_IF_TRUE(if_skip_constructor);
ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END());
jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
} else {
ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]);
/* merge current control path with the true branch of constructor skip condition */
ir_MERGE_WITH_EMPTY_TRUE(if_skip_constructor);
jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END();
jit->b = -1;
}
return 1;
}
static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
ir_ref ref, fast_path = IR_UNUSED;
ref = jit_ZVAL_ADDR(jit, res_addr);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->prev) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
return 1;
}
}
if (ZEND_ARG_SEND_MODE(arg_info)) {
if (opline->opcode == ZEND_RECV_INIT) {
ref = jit_ZVAL_DEREF_ref(jit, ref);
} else {
ref = jit_Z_PTR_ref(jit, ref);
ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
}
}
if (type_mask != 0) {
if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code));
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
} else {
ir_ref if_ok = ir_IF(ir_AND_U32(
ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)),
ir_CONST_U32(type_mask)));
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
}
}
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_verify_arg_slow),
ref, ir_CONST_ADDR(arg_info));
if (check_exception) {
ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler));
}
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
return 1;
}
static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array)
{
uint32_t arg_num = opline->op1.num;
zend_arg_info *arg_info = NULL;
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
if (EXPECTED(arg_num <= op_array->num_args)) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
arg_info = &op_array->arg_info[op_array->num_args];
}
if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
arg_info = NULL;
}
}
if (arg_info || (opline+1)->opcode != ZEND_RECV) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)),
ir_CONST_ADDR(exit_addr));
}
} else {
ir_ref if_ok =ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
ir_IF_FALSE_cold(if_ok);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_TRUE(if_ok);
}
}
if (arg_info) {
if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) {
return 0;
}
}
return 1;
}
static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
{
uint32_t arg_num = opline->op1.num;
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ir_ref ref, if_fail, skip_path = IR_UNUSED;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
jit_ZVAL_COPY_CONST(jit,
res_addr,
-1, -1,
zv, 1);
}
} else {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
ir_ref if_skip = ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)));
ir_IF_TRUE(if_skip);
skip_path = ir_END();
ir_IF_FALSE(if_skip);
}
jit_ZVAL_COPY_CONST(jit,
res_addr,
-1, -1,
zv, 1);
}
if (Z_CONSTANT_P(zv)) {
jit_SET_EX_OPLINE(jit, opline);
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zval_update_constant_ex),
jit_ZVAL_ADDR(jit, res_addr),
ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), offsetof(zend_op_array, scope))));
if_fail = ir_IF(ref);
ir_IF_TRUE_cold(if_fail);
jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_FALSE(if_fail);
}
if (skip_path) {
ir_MERGE_WITH(skip_path);
}
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
do {
zend_arg_info *arg_info;
if (arg_num <= op_array->num_args) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
arg_info = &op_array->arg_info[op_array->num_args];
} else {
break;
}
if (!ZEND_TYPE_IS_SET(arg_info->type)) {
break;
}
if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) {
return 0;
}
} while (0);
}
return 1;
}
static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
{
zend_arg_info *arg_info = &op_array->arg_info[-1];
ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
zend_jit_addr op1_addr = OP1_ADDR();
bool needs_slow_check = 1;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
ir_ref fast_path = IR_UNUSED;
if (type_mask != 0) {
if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
/* pass */
} else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
needs_slow_check = 0;
} else if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code);
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
} else {
ir_ref if_ok = ir_IF(ir_AND_U32(
ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)),
ir_CONST_U32(type_mask)));
ir_IF_TRUE(if_ok);
fast_path = ir_END();
ir_IF_FALSE_cold(if_ok);
}
}
if (needs_slow_check) {
ir_ref ref;
jit_SET_EX_OPLINE(jit, opline);
ref = jit_ZVAL_ADDR(jit, op1_addr);
if (op1_info & MAY_BE_UNDEF) {
ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1);
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_verify_return_slow),
ref,
ir_LOAD_A(jit_EX(func)),
ir_CONST_ADDR(arg_info),
ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->op2.num));
zend_jit_check_exception(jit);
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
}
return 1;
}
static int zend_jit_leave_frame(zend_jit_ctx *jit)
{
// JIT: EG(current_execute_data) = EX(prev_execute_data);
ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
return 1;
}
static int zend_jit_free_cvs(zend_jit_ctx *jit)
{
// JIT: EG(current_execute_data) = EX(prev_execute_data);
ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data)));
// JIT: zend_free_compiled_variables(execute_data);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit));
return 1;
}
static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL);
}
return 1;
}
static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_offset)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline);
}
return 1;
}
static int zend_jit_leave_func(zend_jit_ctx *jit,
const zend_op_array *op_array,
const zend_op *opline,
uint32_t op1_info,
bool left_frame,
zend_jit_trace_rec *trace,
zend_jit_trace_info *trace_info,
int indirect_var_access,
int may_throw)
{
bool may_be_top_frame =
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
bool may_need_call_helper =
indirect_var_access || /* may have symbol table */
!op_array->function_name || /* may have symbol table */
may_be_top_frame ||
(op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
(uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
bool may_need_release_this =
!(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
op_array->scope &&
!(op_array->fn_flags & ZEND_ACC_STATIC) &&
(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
ir_ref call_info = IR_UNUSED, ref, cold_path = IR_UNUSED;
if (may_need_call_helper) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(jit)) {
return 0;
}
}
/* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
ref = ir_AND_U32(call_info,
ir_CONST_U32(ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE));
if (trace && trace->op != ZEND_JIT_TRACE_END) {
ir_ref if_slow = ir_IF(ref);
ir_IF_TRUE_cold(if_slow);
if (!GCC_GLOBAL_REGS) {
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit));
} else {
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_func_helper));
}
if (may_be_top_frame) {
// TODO: try to avoid this check ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
/* this check should be handled by the following OPLINE guard */
| cmp IP, zend_jit_halt_op
| je ->trace_halt
#endif
} else if (GCC_GLOBAL_REGS) {
ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
} else {
ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
}
}
if (!GCC_GLOBAL_REGS) {
// execute_data = EG(current_execute_data)
jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data)));
}
cold_path = ir_END();
ir_IF_FALSE(if_slow);
} else {
ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler));
}
}
if ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) == ZEND_ACC_CLOSURE) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(jit)) {
return 0;
}
}
// JIT: OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object)));
} else if (may_need_release_this) {
ir_ref if_release, fast_path = IR_UNUSED;
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(jit)) {
return 0;
}
}
if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) {
// JIT: if (call_info & ZEND_CALL_RELEASE_THIS)
if (!call_info) {
call_info = ir_LOAD_U32(jit_EX(This.u1.type_info));
}
if_release = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_RELEASE_THIS)));
ir_IF_FALSE(if_release);
fast_path = ir_END();
ir_IF_TRUE(if_release);
}
// JIT: OBJ_RELEASE(execute_data->This))
jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj)));
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
// TODO: avoid EG(excption) check for $this->foo() calls
may_throw = 1;
}
// JIT: EG(vm_stack_top) = (zval*)execute_data
ir_STORE(jit_EG(vm_stack_top), jit_FP(jit));
// JITL execute_data = EX(prev_execute_data)
jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data)));
if (!left_frame) {
// JIT: EG(current_execute_data) = execute_data
ir_STORE(jit_EG(current_execute_data), jit_FP(jit));
}
if (trace) {
if (trace->op != ZEND_JIT_TRACE_END
&& (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
zend_jit_reset_last_valid_opline(jit);
} else {
if (GCC_GLOBAL_REGS) {
/* We add extra RLOAD and RSTORE to make fusion for persistent register
* mov (%FP), %IP
* add $0x1c, %IP
* The naive (commented) code leads to extra register allocation and move.
* mov (%FP), %tmp
* add $0x1c, %tmp
* mov %tmp, %FP
*/
#if 0
jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op)));
#else
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
#endif
} else {
ir_ref ref = jit_EX(opline);
ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
}
}
if (cold_path) {
ir_MERGE_WITH(cold_path);
}
if (trace->op == ZEND_JIT_TRACE_BACK
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
const zend_op *next_opline = trace->opline;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
/* exception might be thrown during destruction of unused return value */
// JIT: if (EG(exception))
ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
}
do {
trace++;
} while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
next_opline = trace->opline;
ZEND_ASSERT(next_opline != NULL);
if (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
trace_info->flags |= ZEND_JIT_TRACE_LOOP;
ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline));
ir_IF_TRUE(if_eq);
ZEND_ASSERT(jit->trace_loop_ref);
ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED);
ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END());
ir_IF_FALSE(if_eq);
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
#else
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape));
#endif
} else {
ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape));
}
zend_jit_set_last_valid_opline(jit, trace->opline);
return 1;
} else if (may_throw ||
(((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
// JIT: if (EG(exception))
ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
}
return 1;
} else {
// JIT: if (EG(exception))
ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw));
// JIT: opline = EX(opline) + 1
if (GCC_GLOBAL_REGS) {
jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline)));
jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op)));
} else {
ir_ref ref = jit_EX(opline);
ir_STORE(ref, ir_ADD_OFFSET(ir_LOAD_A(ref), sizeof(zend_op)));
}
}
if (GCC_GLOBAL_REGS) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
}
jit->b = -1;
return 1;
}
static void zend_jit_common_return(zend_jit_ctx *jit)
{
ZEND_ASSERT(jit->return_inputs);
ir_MERGE_list(jit->return_inputs);
}
static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
{
zend_jit_addr ret_addr;
int8_t return_value_used = -1;
ir_ref return_value = IR_UNUSED, ref, refcount, if_return_value_used = IR_UNUSED;
ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
jit->return_inputs = IR_UNUSED;
if (JIT_G(current_frame)) {
if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
return_value_used = 1;
} else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
return_value_used = 0;
} else {
return_value_used = -1;
}
}
}
if (ZEND_OBSERVER_ENABLED) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) {
return 0;
}
op1_addr = dst;
}
jit_observer_fcall_end(jit, jit_FP(jit), jit_ZVAL_ADDR(jit, op1_addr));
}
// JIT: if (!EX(return_value))
return_value = ir_LOAD_A(jit_EX(return_value));
ret_addr = ZEND_ADDR_REF_ZVAL(return_value);
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (return_value_used == -1) {
if_return_value_used = ir_IF(return_value);
ir_IF_FALSE_cold(if_return_value_used);
}
if (return_value_used != 1) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(jit->return_inputs);
ir_IF_TRUE(if_refcounted);
}
ref = jit_Z_PTR(jit, op1_addr);
refcount = jit_GC_DELREF(jit, ref);
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
ir_ref if_non_zero = ir_IF(refcount);
ir_IF_TRUE(if_non_zero);
ir_END_list(jit->return_inputs);
ir_IF_FALSE(if_non_zero);
}
jit_ZVAL_DTOR(jit, ref, op1_info, opline);
}
if (return_value_used == -1) {
ir_END_list(jit->return_inputs);
}
}
} else if (return_value_used == -1) {
if_return_value_used = ir_IF(return_value);
ir_IF_FALSE_cold(if_return_value_used);
ir_END_list(jit->return_inputs);
}
if (if_return_value_used) {
ir_IF_TRUE(if_return_value_used);
}
if (return_value_used == 0) {
if (jit->return_inputs) {
ZEND_ASSERT(JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE);
ir_END_list(jit->return_inputs);
ir_MERGE_list(jit->return_inputs);
jit->return_inputs = IR_UNUSED;
}
return 1;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
} else if (opline->op1_type == IS_TMP_VAR) {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_REF) {
ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
!op_array->function_name) {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1);
} else if (return_value_used != 1) {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
// JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL);
} else {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
}
} else {
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
}
} else {
if (op1_info & MAY_BE_REF) {
ir_ref if_ref, ref2, if_non_zero;
zend_jit_addr ref_addr;
if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_TRUE_cold(if_ref);
// JIT: zend_refcounted *ref = Z_COUNTED_P(retval_ptr)
ref = jit_Z_PTR(jit, op1_addr);
// JIT: ZVAL_COPY_VALUE(return_value, &ref->value)
ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
ref_addr = ZEND_ADDR_REF_ZVAL(ref2);
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0);
ref2 = jit_GC_DELREF(jit, ref);
if_non_zero = ir_IF(ref2);
ir_IF_TRUE(if_non_zero);
// JIT: if (IS_REFCOUNTED())
ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr);
ir_IF_FALSE(if_refcounted);
ir_END_list(jit->return_inputs);
ir_IF_TRUE(if_refcounted);
// JIT: ADDREF
ref2 = jit_Z_PTR(jit, ret_addr);
jit_GC_ADDREF(jit, ref2);
ir_END_list(jit->return_inputs);
ir_IF_FALSE(if_non_zero);
jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline);
ir_END_list(jit->return_inputs);
ir_IF_FALSE(if_ref);
}
jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (jit->return_inputs) {
ir_END_list(jit->return_inputs);
ir_MERGE_list(jit->return_inputs);
jit->return_inputs = IR_UNUSED;
}
} else {
ir_END_list(jit->return_inputs);
jit->b = -1;
}
return 1;
}
static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr op1_addr = OP1_ADDR();
zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
ir_ref cache_slot_ref, idx_ref, num_used_ref, bucket_ref, ref, ref2;
ir_ref if_fit, if_reference, if_same_key, fast_path;
ir_ref slow_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
// JIT: idx = (uintptr_t)CACHED_PTR(opline->extended_value) - 1;
cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value);
idx_ref = ir_SUB_A(ir_LOAD_A(cache_slot_ref), ir_CONST_ADDR(1));
// JIT: if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
num_used_ref = ir_MUL_U32(ir_LOAD_U32(jit_EG(symbol_table.nNumUsed)),
ir_CONST_U32(sizeof(Bucket)));
if (sizeof(void*) == 8) {
num_used_ref = ir_ZEXT_A(num_used_ref);
}
if_fit = ir_IF(ir_ULT(idx_ref, num_used_ref));
ir_IF_FALSE_cold(if_fit);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_fit);
// JIT: Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
bucket_ref = ir_ADD_A(ir_LOAD_A(jit_EG(symbol_table.arData)), idx_ref);
if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE));
ir_IF_FALSE_cold(if_reference);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_reference);
// JIT: (EXPECTED(p->key == varname))
if_same_key = ir_IF(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(bucket_ref, offsetof(Bucket, key))), ir_CONST_ADDR(varname)));
ir_IF_FALSE_cold(if_same_key);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same_key);
// JIT: GC_ADDREF(Z_PTR(p->val))
ref = jit_Z_PTR_ref(jit, bucket_ref);
jit_GC_ADDREF(jit, ref);
fast_path = ir_END();
ir_MERGE_list(slow_inputs);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_global_helper),
ir_CONST_ADDR(varname),
cache_slot_ref);
ir_MERGE_WITH(fast_path);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
ir_ref if_refcounted = IR_UNUSED, refcount, if_non_zero, if_may_not_leak;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
// JIT: if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
if_refcounted = jit_if_REFCOUNTED(jit, op1_addr);
ir_IF_TRUE_cold(if_refcounted);
}
// JIT:zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
ref2 = jit_Z_PTR(jit, op1_addr);
// JIT: ZVAL_REF(variable_ptr, ref)
jit_set_Z_PTR(jit, op1_addr, ref);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
// JIT: if (GC_DELREF(garbage) == 0)
refcount = jit_GC_DELREF(jit, ref2);
if_non_zero = ir_IF(refcount);
if (!(op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
ir_IF_TRUE(if_non_zero);
ir_END_list(end_inputs);
}
ir_IF_FALSE(if_non_zero);
jit_ZVAL_DTOR(jit, ref2, op1_info, opline);
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
ir_END_list(end_inputs);
ir_IF_TRUE(if_non_zero);
// JIT: GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2);
ir_IF_TRUE(if_may_not_leak);
ir_END_list(end_inputs);
ir_IF_FALSE(if_may_not_leak);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref2);
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
ir_END_list(end_inputs);
ir_IF_FALSE(if_refcounted);
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
// JIT: ZVAL_REF(variable_ptr, ref)
jit_set_Z_PTR(jit, op1_addr, ref);
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX);
}
if (end_inputs) {
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw)
{
zend_jit_addr op1_addr = OP1_ADDR();
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (may_throw) {
jit_SET_EX_OPLINE(jit, opline);
}
if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
ir_ref ref, if_array, if_exists, end_inputs = IR_UNUSED;
if (op1_info & MAY_BE_ARRAY) {
if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(if_array);
ir_END_list(end_inputs);
ir_IF_FALSE(if_array);
}
ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx)));
if_exists = ir_IF(ir_EQ(ref, ir_CONST_U32(-1)));
ir_IF_TRUE(if_exists);
ir_END_list(end_inputs);
ir_IF_FALSE(if_exists);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_hash_iterator_del), ref);
ir_END_list(end_inputs);
ir_MERGE_list(end_inputs);
}
jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
}
return 1;
}
static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
if (len > 0) {
const char *str = Z_STRVAL_P(zv);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
ir_CONST_ADDR(str), ir_CONST_ADDR(len));
zend_jit_check_exception(jit);
}
} else {
zend_jit_addr op1_addr = OP1_ADDR();
ir_ref ref;
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
jit_SET_EX_OPLINE(jit, opline);
ref = jit_Z_PTR(jit, op1_addr);
ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write),
ir_ADD_OFFSET(ref, offsetof(zend_string, val)),
ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_string, len))));
if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline);
}
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len));
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
} else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ir_ref ref;
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
ref = jit_Z_PTR(jit, op1_addr);
ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(zend_string, len)));
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) == IS_REG) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
return 1;
}
static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
zend_long count;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
count = zend_hash_num_elements(Z_ARRVAL_P(zv));
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count));
if (Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
} else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ir_ref ref;
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
// Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.
ref = jit_Z_PTR(jit, op1_addr);
if (sizeof(void*) == 8) {
ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
ref = ir_ZEXT_L(ref);
} else {
ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements)));
}
jit_set_Z_LVAL(jit, res_addr, ref);
if (Z_MODE(res_addr) == IS_REG) {
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ir_ref ref;
ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
// JIT: result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
if (opline->op1_type != IS_CONST) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find),
ir_CONST_ADDR(ht),
jit_Z_PTR(jit, op1_addr));
} else {
zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash),
ir_CONST_ADDR(ht), ir_CONST_ADDR(str));
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
} else if (smart_branch_opcode) {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
ZEND_ASSERT(bb->successors_count == 2);
ref = jit_IF_ex(jit, ref,
(smart_branch_opcode == ZEND_JMPZ) ? target_label2 : target_label);
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref);
jit->b = -1;
} else {
jit_set_Z_TYPE_INFO_ex(jit, res_addr,
ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, IR_NULL)), ir_CONST_U32(IS_FALSE)));
}
return 1;
}
static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info)
{
uint32_t offset;
offset = (opline->opcode == ZEND_ROPE_INIT) ?
opline->result.var :
opline->op1.var + opline->extended_value * sizeof(zend_string*);
if (opline->op2_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_string *str;
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
str = Z_STR_P(zv);
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str));
} else {
zend_jit_addr op2_addr = OP2_ADDR();
ir_ref ref;
ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
ref = jit_Z_PTR(jit, op2_addr);
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref);
if (opline->op2_type == IS_CV) {
ir_ref if_refcounted, long_path;
if_refcounted = jit_if_REFCOUNTED(jit, op2_addr);
ir_IF_TRUE(if_refcounted);
jit_GC_ADDREF(jit, ref);
long_path = ir_END();
ir_IF_FALSE(if_refcounted);
ir_MERGE_WITH(long_path);
}
}
if (opline->opcode == ZEND_ROPE_END) {
zend_jit_addr res_addr = RES_ADDR();
ir_ref ref;
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_rope_end),
ir_ADD_OFFSET(jit_FP(jit), opline->op1.var),
ir_CONST_U32(opline->extended_value));
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
}
return 1;
}
static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_addr, ir_ref type)
{
ir_ref if_refcounted, if_reference, if_refcounted2, ptr, val2, ptr2, type2;
ir_refs *merge_inputs, *types, *ptrs;
#if SIZEOF_ZEND_LONG == 4
ir_ref val = jit_ZVAL_ADDR(jit, val_addr);
ir_refs *values; /* we need this only for zval.w2 copy */
#endif
ir_refs_init(merge_inputs, 4);
ir_refs_init(types, 4);
ir_refs_init(ptrs, 4);
#if SIZEOF_ZEND_LONG == 4
ir_refs_init(values, 4);
#endif
// JIT: ptr = Z_PTR_P(val);
ptr = jit_Z_PTR(jit, val_addr);
// JIT: if (Z_OPT_REFCOUNTED_P(val)) {
if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
ir_IF_FALSE_cold(if_refcounted);
ir_refs_add(merge_inputs, ir_END());
ir_refs_add(types, type);
ir_refs_add(ptrs, ptr);
#if SIZEOF_ZEND_LONG == 4
ir_refs_add(values, val);
#endif
ir_IF_TRUE(if_refcounted);
// JIT: if (UNEXPECTED(Z_OPT_ISREF_P(val))) {
if_reference = ir_IF(ir_EQ(type, ir_CONST_U32(IS_REFERENCE_EX)));
// if_reference = ir_IF(ir_EQ(ir_TRUNC_U8(type), ir_CONST_U8(IS_REFERENCE))); // TODO: fix IR to avoid need for extra register ???
ir_IF_TRUE(if_reference);
// JIT: val = Z_REFVAL_P(val);
val2 = ir_ADD_OFFSET(ptr, offsetof(zend_reference, val));
type2 = jit_Z_TYPE_INFO_ref(jit, val2);
ptr2 = jit_Z_PTR_ref(jit, val2);
// JIT: if (Z_OPT_REFCOUNTED_P(val)) {
if_refcounted2 = ir_IF(ir_AND_U32(type2, ir_CONST_U32(Z_TYPE_FLAGS_MASK)));
ir_IF_FALSE_cold(if_refcounted2);
ir_refs_add(merge_inputs, ir_END());
ir_refs_add(types, type2);
ir_refs_add(ptrs, ptr2);
#if SIZEOF_ZEND_LONG == 4
ir_refs_add(values, val2);
#endif
ir_IF_TRUE(if_refcounted2);
ir_MERGE_WITH_EMPTY_FALSE(if_reference);
type = ir_PHI_2(IR_U32, type2, type);
ptr = ir_PHI_2(IR_ADDR, ptr2, ptr);
#if SIZEOF_ZEND_LONG == 4
val = ir_PHI_2(IR_ADDR, val2, val);
#endif
// JIT: Z_ADDREF_P(val);
jit_GC_ADDREF(jit, ptr);
ir_refs_add(merge_inputs, ir_END());
ir_refs_add(types, type);
ir_refs_add(ptrs, ptr);
#if SIZEOF_ZEND_LONG == 4
ir_refs_add(values, val);
#endif
ir_MERGE_N(merge_inputs->count, merge_inputs->refs);
type = ir_PHI_N(IR_U32, types->count, types->refs);
ptr = ir_PHI_N(IR_ADDR, ptrs->count, ptrs->refs);
#if SIZEOF_ZEND_LONG == 4
val = ir_PHI_N(IR_ADDR, values->count, values->refs);
val_addr = ZEND_ADDR_REF_ZVAL(val);
#endif
// JIT: Z_PTR_P(res) = ptr;
jit_set_Z_PTR(jit, res_addr, ptr);
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr));
#endif
jit_set_Z_TYPE_INFO_ex(jit, res_addr, type);
return 1;
}
static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t type,
uint32_t op1_info,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint8_t dim_type,
const void *found_exit_addr,
const void *not_found_exit_addr,
const void *exit_addr,
bool result_type_guard,
ir_ref ht_ref,
ir_refs *found_inputs,
ir_refs *found_vals,
ir_ref *end_inputs,
ir_ref *not_found_inputs)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ir_ref ref = IR_UNUSED, cond, if_found;
ir_ref if_type = IS_UNUSED;
ir_refs *test_zval_inputs, *test_zval_values;
ir_refs_init(test_zval_inputs, 4);
ir_refs_init(test_zval_values, 4);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& type == BP_VAR_R
&& !exit_addr) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (op2_info & MAY_BE_LONG) {
bool op2_loaded = 0;
bool packed_loaded = 0;
bool bad_packed_key = 0;
ir_ref if_packed = IS_UNDEF;
ir_ref h = IR_UNUSED;
ir_ref idx_not_found_inputs = IR_UNUSED;
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
// JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG);
ir_IF_TRUE(if_type);
}
if (op1_info & MAY_BE_PACKED_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
cond = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED));
if (op1_info & MAY_BE_ARRAY_PACKED) {
ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr));
}
}
if (type == BP_VAR_W) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
op2_loaded = 1;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
zend_long val = -1;
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
val = Z_LVAL_P(Z_ZV(op2_addr));
if (val >= 0 && val < HT_MAX_SIZE) {
packed_loaded = 1;
} else {
bad_packed_key = 1;
}
h = ir_CONST_LONG(val);
} else {
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
op2_loaded = 1;
}
packed_loaded = 1;
}
if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) {
/* don't generate "fast" code for packed array */
packed_loaded = 0;
}
if (packed_loaded) {
// JIT: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if_packed = ir_IF(
ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED)));
ir_IF_TRUE(if_packed);
}
// JIT: if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
ref = ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed)));
#if SIZEOF_ZEND_LONG == 8
if ((Z_MODE(op2_addr) == IS_CONST_ZVAL && val >= 0 && val <= UINT32_MAX)
|| (op2_range && op2_range->min >= 0 && op2_range->max <= UINT32_MAX)) {
/* comapre only the lower 32-bits to allow load fusion on x86_64 */
cond = ir_ULT(ir_TRUNC_U32(h), ref);
} else {
cond = ir_ULT(h, ir_ZEXT_L(ref));
}
#else
cond = ir_ULT(h, ref);
#endif
if (type == BP_JIT_IS) {
if (not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
} else {
ir_ref if_fit = ir_IF(cond);
ir_IF_FALSE(if_fit);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_fit);
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(cond, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_RW && not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
ir_ref if_fit = ir_IF(cond);
ir_IF_FALSE(if_fit);
ir_END_list(*not_found_inputs);
ir_IF_TRUE(if_fit);
} else {
ir_ref if_fit = ir_IF(cond);
ir_IF_FALSE(if_fit);
ir_END_list(idx_not_found_inputs);
ir_IF_TRUE(if_fit);
}
// JIT: _ret = &_ht->arPacked[h];
ref = ir_MUL_L(h, ir_CONST_LONG(sizeof(zval)));
ref = ir_BITCAST_A(ref);
ref = ir_ADD_A(ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))), ref);
if (type == BP_JIT_IS) {
ir_refs_add(test_zval_values, ref);
ir_refs_add(test_zval_inputs, ir_END());
}
}
}
switch (type) {
case BP_JIT_IS:
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
}
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
op2_loaded = 1;
}
if (packed_loaded) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(test_zval_values, ref);
ir_refs_add(test_zval_inputs, ir_END());
} else if (!not_found_exit_addr && !packed_loaded) {
ir_END_list(*end_inputs);
}
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (packed_loaded) {
ir_ref type_ref = jit_Z_TYPE_ref(jit, ref);
if (result_type_guard) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(type_ref, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(type_ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
ir_ref if_def = ir_IF(type_ref);
ir_IF_FALSE(if_def);
ir_END_list(idx_not_found_inputs);
ir_IF_TRUE(if_def);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
}
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
}
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
op2_loaded = 1;
}
if (packed_loaded) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*not_found_inputs);
ir_IF_TRUE(if_found);
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(idx_not_found_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
} else if (!packed_loaded) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
ir_END_list(*not_found_inputs);
} else {
ir_END_list(idx_not_found_inputs);
}
}
if (idx_not_found_inputs) {
ir_MERGE_list(idx_not_found_inputs);
switch (type) {
case BP_VAR_R:
ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE);
// JIT: zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
// JIT: retval = &EG(uninitialized_zval);
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(op2_addr) == IS_REG) {
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
}
if (GCC_GLOBAL_REGS) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h);
} else {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h, jit_FP(jit));
}
} else {
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC));
}
ir_END_list(*end_inputs);
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
if (!not_found_exit_addr) {
// JIT: retval = &EG(uninitialized_zval);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(*end_inputs);
}
break;
default:
ZEND_UNREACHABLE();
}
}
break;
case BP_VAR_RW:
if (packed_loaded) {
if (not_found_exit_addr) {
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
ir_IF_TRUE(if_def);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE_cold(if_def);
ir_END_list(idx_not_found_inputs);
}
}
if (!packed_loaded ||
!not_found_exit_addr ||
(op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
ir_END_list(idx_not_found_inputs);
} else if (!packed_loaded) {
ir_END_list(idx_not_found_inputs);
}
ir_MERGE_list(idx_not_found_inputs);
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
}
if (packed_loaded) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw_no_packed),
ht_ref, h);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw), ht_ref, h);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
}
break;
case BP_VAR_W:
if (packed_loaded) {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
ir_IF_TRUE_cold(if_def);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE(if_def);
ir_END_list(idx_not_found_inputs);
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
if (if_packed) {
ir_IF_FALSE(if_packed);
if_packed = IR_UNUSED;
ir_END_list(idx_not_found_inputs);
} else if (!packed_loaded) {
ir_END_list(idx_not_found_inputs);
}
ir_MERGE_list(idx_not_found_inputs);
if (!op2_loaded) {
// JIT: hval = Z_LVAL_P(dim);
h = jit_Z_LVAL(jit, op2_addr);
}
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_lookup), ht_ref, h);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (op2_info & MAY_BE_STRING) {
ir_ref key;
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IS_UNUSED;
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
// JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING);
ir_IF_TRUE(if_type);
}
// JIT: offset_key = Z_STR_P(dim);
key = jit_Z_PTR(jit, op2_addr);
// JIT: retval = zend_hash_find(ht, offset_key);
switch (type) {
case BP_JIT_IS:
if (opline->op2_type != IS_CONST) {
ir_ref if_num, end1, ref2;
if_num = ir_IF(
ir_ULE(
ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
ir_CONST_CHAR('9')));
ir_IF_TRUE_cold(if_num);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
end1 = ir_END();
ir_IF_FALSE(if_num);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(test_zval_values, ref);
ir_refs_add(test_zval_inputs, ir_END());
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (opline->op2_type != IS_CONST) {
ir_ref if_num, end1, ref2;
if_num = ir_IF(
ir_ULE(
ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))),
ir_CONST_CHAR('9')));
ir_IF_TRUE_cold(if_num);
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key);
end1 = ir_END();
ir_IF_FALSE(if_num);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key);
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else if (type == BP_VAR_IS && not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else if (type == BP_VAR_IS && result_type_guard) {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*not_found_inputs);
ir_IF_TRUE(if_found);
} else {
if_found = ir_IF(ref);
switch (type) {
case BP_VAR_R:
ir_IF_FALSE_cold(if_found);
// JIT: zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
jit_SET_EX_OPLINE(jit, opline);
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC));
ir_END_list(*end_inputs);
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
ir_IF_FALSE(if_found);
// JIT: retval = &EG(uninitialized_zval);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(*end_inputs);
break;
default:
ZEND_UNREACHABLE();
}
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
break;
case BP_VAR_RW:
if (opline->op2_type != IS_CONST) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_rw), ht_ref, key);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_lookup_rw), ht_ref, key);
}
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
} else {
if_found = ir_IF(ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_found);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
break;
case BP_VAR_W:
if (opline->op2_type != IS_CONST) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_w), ht_ref, key);
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_lookup), ht_ref, key);
}
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
break;
default:
ZEND_UNREACHABLE();
}
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
if (type != BP_VAR_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
ref = jit_ZVAL_ADDR(jit, op2_addr);
switch (type) {
case BP_VAR_R:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_r_helper),
ht_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(*end_inputs);
break;
case BP_JIT_IS:
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_fetch_dim_isset_helper), ht_ref, ref);
if (not_found_exit_addr) {
ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr));
ir_refs_add(found_inputs, ir_END());
} else if (found_exit_addr) {
ir_GUARD_NOT(ref, ir_CONST_ADDR(found_exit_addr));
ir_END_list(*end_inputs);
} else {
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
ir_refs_add(found_inputs, ir_END());
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_is_helper),
ht_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(*end_inputs);
break;
case BP_VAR_RW:
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_rw_helper), ht_ref, ref);
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
break;
case BP_VAR_W:
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_w_helper), ht_ref, ref);
if_found = ir_IF(ref);
ir_IF_TRUE(if_found);
ir_refs_add(found_inputs, ir_END());
ir_refs_add(found_vals, ref);
ir_IF_FALSE(if_found);
ir_END_list(*end_inputs);
break;
default:
ZEND_UNREACHABLE();
}
}
if (type == BP_JIT_IS
&& !(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
/* dead code */
ir_END_list(*end_inputs);
} else if (type == BP_JIT_IS
&& (op1_info & MAY_BE_ARRAY)
&& (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
&& test_zval_inputs->count) {
ir_MERGE_N(test_zval_inputs->count, test_zval_inputs->refs);
ref = ir_PHI_N(IR_ADDR, test_zval_values->count, test_zval_values->refs);
if (op1_info & MAY_BE_ARRAY_OF_REF) {
ref = jit_ZVAL_DEREF_ref(jit, ref);
}
cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL));
if (not_found_exit_addr) {
ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr));
ir_refs_add(found_inputs, ir_END());
} else if (found_exit_addr) {
ir_GUARD_NOT(cond, ir_CONST_ADDR(found_exit_addr));
ir_END_list(*end_inputs);
} else {
ir_ref if_set = ir_IF(cond);
ir_IF_FALSE(if_set);
ir_END_list(*end_inputs);
ir_IF_TRUE(if_set);
ir_refs_add(found_inputs, ir_END());
}
}
return 1;
}
static int zend_jit_fetch_dim_read(zend_jit_ctx *jit,
const zend_op *opline,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint32_t res_info,
zend_jit_addr res_addr,
uint8_t dim_type)
{
zend_jit_addr orig_op1_addr;
const void *exit_addr = NULL;
const void *not_found_exit_addr = NULL;
bool result_type_guard = 0;
bool result_avoid_refcounting = 0;
uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
int may_throw = 0;
ir_ref if_type = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref not_found_inputs = IR_UNUSED;
orig_op1_addr = OP1_ADDR();
if (opline->opcode != ZEND_FETCH_DIM_IS
&& JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& !has_concrete_type(op1_info)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if ((res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
result_type_guard = 1;
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& (opline->opcode == ZEND_FETCH_LIST_R
|| !(opline->op1_type & (IS_VAR|IS_TMP_VAR))
|| op1_avoid_refcounting)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
if (opline->opcode == ZEND_FETCH_DIM_IS
&& !(res_info & MAY_BE_NULL)) {
uint32_t flags = 0;
uint32_t old_op1_info = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !op1_avoid_refcounting) {
flags |= ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
flags |= ZEND_JIT_EXIT_FREE_OP2;
}
if (op1_avoid_refcounting) {
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
if (op1_avoid_refcounting) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
}
}
}
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & MAY_BE_ARRAY) {
ir_ref ht_ref, ref;
zend_jit_addr val_addr;
ir_refs *found_inputs, *found_vals;
ir_refs_init(found_inputs, 10);
ir_refs_init(found_vals, 10);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr);
} else {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(if_type);
}
}
ht_ref = jit_Z_PTR(jit, op1_addr);
if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
(opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline,
(opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS,
op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, exit_addr,
result_type_guard, ht_ref, found_inputs, found_vals,
&end_inputs, &not_found_inputs)) {
return 0;
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
val_addr = ZEND_ADDR_REF_ZVAL(ref);
if (result_type_guard) {
uint8_t type = concrete_type(res_info);
uint32_t flags = 0;
if (opline->opcode != ZEND_FETCH_LIST_R
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !op1_avoid_refcounting) {
flags |= ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
flags |= ZEND_JIT_EXIT_FREE_OP2;
}
val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
(op1_info & MAY_BE_ARRAY_OF_REF) != 0, flags, op1_avoid_refcounting);
if (!val_addr) {
return 0;
}
if (not_found_inputs) {
ir_END_list(not_found_inputs);
ir_MERGE_list(not_found_inputs);
}
// ZVAL_COPY
jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
if (Z_MODE(res_addr) != IS_REG) {
} else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
} else if (op1_info & MAY_BE_ARRAY_OF_REF) {
// ZVAL_COPY_DEREF
ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr);
if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) {
return 0;
}
} else {
// ZVAL_COPY
jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1);
}
ir_END_list(end_inputs);
} else if (not_found_inputs) {
ir_MERGE_list(not_found_inputs);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(end_inputs);
} else if (!end_inputs && jit->ctx.control) {
ir_END_list(end_inputs); /* dead code */
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
ir_ref str_ref;
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr);
} else {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_type);
}
}
jit_SET_EX_OPLINE(jit, opline);
str_ref = jit_Z_PTR(jit, op1_addr);
if (opline->opcode != ZEND_FETCH_DIM_IS) {
ir_ref ref;
if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_offset_r_helper),
str_ref, jit_Z_LVAL(jit, op2_addr));
} else {
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_r_helper),
str_ref, jit_ZVAL_ADDR(jit, op2_addr));
}
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_is_helper),
str_ref,
jit_ZVAL_ADDR(jit, op2_addr),
jit_ZVAL_ADDR(jit, res_addr));
}
ir_END_list(end_inputs);
}
if (op1_info & MAY_BE_OBJECT) {
ir_ref arg2;
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
if (exit_addr) {
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_TRUE(if_type);
}
}
jit_SET_EX_OPLINE(jit, opline);
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
if (opline->opcode != ZEND_FETCH_DIM_IS) {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_r_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
} else {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_is_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
}
ir_END_list(end_inputs);
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
&& (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNDEF;
}
if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
may_throw = 1;
zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0);
}
if (op2_info & MAY_BE_UNDEF) {
may_throw = 1;
zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0);
}
}
if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
ir_ref ref;
may_throw = 1;
if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
} else {
jit_SET_EX_OPLINE(jit, opline);
ref = jit_ZVAL_ADDR(jit, op1_addr);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_array_access), ref);
}
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_END_list(end_inputs);
}
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetGet() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
}
if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
}
if (may_throw) {
zend_jit_check_exception(jit);
}
} else if (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) {
ir_BEGIN(IR_UNUSED); /* unreachable tail */
}
return 1;
}
static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
ir_ref *if_type,
ir_ref *ht_ref,
int *may_throw)
{
ir_ref ref = IR_UNUSED;
ir_ref array_reference_end = IR_UNUSED, array_reference_ref = IR_UNUSED;
ir_refs *array_inputs, *array_values;
ir_refs_init(array_inputs, 4);
ir_refs_init(array_values, 4);
ref = jit_ZVAL_ADDR(jit, op1_addr);
if (op1_info & MAY_BE_REF) {
ir_ref if_reference, if_array, end1, ref2;
*may_throw = 1;
if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_FALSE(if_reference);
end1 = ir_END();
ir_IF_TRUE_cold(if_reference);
array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val));
if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY));
ir_IF_TRUE(if_array);
array_reference_end = ir_END();
ir_IF_FALSE_cold(if_array);
if (opline->opcode != ZEND_FETCH_DIM_RW && opline->opcode != ZEND_ASSIGN_DIM_OP) {
jit_SET_EX_OPLINE(jit, opline);
}
ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_prepare_assign_dim_ref), ref);
ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
ir_MERGE_WITH(end1);
ref = ir_PHI_2(IR_ADDR, ref2, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & MAY_BE_ARRAY) {
ir_ref op1_ref = ref;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
*if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(*if_type);
}
if (array_reference_end) {
ir_MERGE_WITH(array_reference_end);
op1_ref = ir_PHI_2(IR_ADDR, ref, array_reference_ref);
}
// JIT: SEPARATE_ARRAY()
ref = jit_Z_PTR_ref(jit, op1_ref);
if (RC_MAY_BE_N(op1_info)) {
if (RC_MAY_BE_1(op1_info)) {
ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1)));
ir_IF_TRUE(if_refcount_1);
ir_refs_add(array_inputs, ir_END());
ir_refs_add(array_values, ref);
ir_IF_FALSE(if_refcount_1);
}
ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_zval_array_dup), op1_ref);
}
if (array_inputs->count || (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
ir_refs_add(array_inputs, ir_END());
ir_refs_add(array_values, ref);
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (*if_type) {
ir_IF_FALSE_cold(*if_type);
*if_type = IR_UNUSED;
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
*if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)));
ir_IF_TRUE(*if_type);
}
if ((op1_info & MAY_BE_UNDEF)
&& (opline->opcode == ZEND_FETCH_DIM_RW || opline->opcode == ZEND_ASSIGN_DIM_OP)) {
ir_ref end1 = IR_UNUSED;
*may_throw = 1;
if (op1_info & MAY_BE_NULL) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
end1 = ir_END();
ir_IF_FALSE(if_def);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var));
if (end1) {
ir_MERGE_WITH(end1);
}
}
// JIT: ZVAL_ARR(container, zend_new_array(8));
ref = ir_CALL_1(IR_ADDR,
jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC),
jit_ZVAL_ADDR(jit, op1_addr));
if (array_inputs->count) {
ir_refs_add(array_inputs, ir_END());
ir_refs_add(array_values, ref);
}
}
if (array_inputs->count) {
ir_MERGE_N(array_inputs->count, array_inputs->refs);
ref = ir_PHI_N(IR_ADDR, array_values->count, array_values->refs);
}
*ht_ref = ref;
return op1_addr;
}
static int zend_jit_fetch_dim(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
zend_jit_addr res_addr,
uint8_t dim_type)
{
int may_throw = 0;
ir_ref end_inputs = IR_UNUSED;
ir_ref ref, if_type = IR_UNUSED, ht_ref;
if (opline->opcode == ZEND_FETCH_DIM_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
ir_refs *found_inputs, *found_vals;
ir_refs_init(found_inputs, 8);
ir_refs_init(found_vals, 8);
if (opline->op2_type == IS_UNUSED) {
ir_ref if_ok;
may_throw = 1;
// JIT:var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
ht_ref, jit_EG(uninitialized_zval));
// JIT: if (UNEXPECTED(!var_ptr)) {
if_ok = ir_IF(ref);
ir_IF_FALSE_cold(if_ok);
if (opline->opcode != ZEND_FETCH_DIM_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
ir_END_list(end_inputs);
ir_IF_TRUE(if_ok);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
ir_END_list(end_inputs);
} else {
uint32_t type;
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
type = BP_VAR_W;
break;
case ZEND_FETCH_DIM_RW:
may_throw = 1;
type = BP_VAR_RW;
break;
case ZEND_FETCH_DIM_UNSET:
type = BP_VAR_UNSET;
break;
default:
ZEND_UNREACHABLE();
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info,
op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
0, ht_ref, found_inputs, found_vals, &end_inputs, NULL)) {
return 0;
}
if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
if (end_inputs) {
ir_MERGE_list(end_inputs);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
end_inputs = ir_END();
}
} else if (!(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
/* impossible dead path */
end_inputs = ir_END();
} else {
ZEND_ASSERT(end_inputs == IR_UNUSED);
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
ir_END_list(end_inputs);
}
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
ir_ref arg2;
may_throw = 1;
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IR_UNUSED;
}
if (opline->opcode != ZEND_FETCH_DIM_RW) {
jit_SET_EX_OPLINE(jit, opline);
}
if (opline->op2_type == IS_UNUSED) {
arg2 = IR_NULL;
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_w_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
break;
case ZEND_FETCH_DIM_RW:
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_rw_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, res_addr));
break;
// case ZEND_FETCH_DIM_UNSET:
// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0
// break;
default:
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
ir_END_list(end_inputs);
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & MAY_HAVE_DTOR)
&& (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint8_t dim_type,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr)
{
zend_jit_addr res_addr;
ir_ref if_type = IR_UNUSED;
ir_ref false_inputs = IR_UNUSED, end_inputs = IR_UNUSED;
ir_refs *true_inputs;
ir_refs_init(true_inputs, 8);
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (op1_info & MAY_BE_REF) {
ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr);
ref = jit_ZVAL_DEREF_ref(jit, ref);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
}
if (op1_info & MAY_BE_ARRAY) {
const void *found_exit_addr = NULL;
const void *not_found_exit_addr = NULL;
ir_ref ht_ref;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY);
ir_IF_TRUE(if_type);
}
ht_ref = jit_Z_PTR(jit, op1_addr);
if (exit_addr
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
&& !may_throw
&& (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
&& (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
if (smart_branch_opcode == ZEND_JMPNZ) {
found_exit_addr = exit_addr;
} else {
not_found_exit_addr = exit_addr;
}
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info,
op2_info, op2_addr, op2_range, dim_type, found_exit_addr, not_found_exit_addr, NULL,
0, ht_ref, true_inputs, NULL, &false_inputs, NULL)) {
return 0;
}
if (found_exit_addr) {
ir_MERGE_list(false_inputs);
return 1;
} else if (not_found_exit_addr) {
ir_MERGE_N(true_inputs->count, true_inputs->refs);
return 1;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IR_UNUSED;
}
if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
ir_ref ref, arg1, arg2, if_true;
jit_SET_EX_OPLINE(jit, opline);
arg1 = jit_ZVAL_ADDR(jit, op1_addr);
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_isset_dim_helper), arg1, arg2);
if_true = ir_IF(ref);
ir_IF_TRUE(if_true);
ir_refs_add(true_inputs, ir_END());
ir_IF_FALSE(if_true);
ir_END_list(false_inputs);
} else {
if (op2_info & MAY_BE_UNDEF) {
ir_ref end1 = IR_UNUSED;
if (op2_info & MAY_BE_ANY) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr));
ir_IF_TRUE(if_def);
end1 = ir_END();
ir_IF_FALSE(if_def);
}
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var));
if (end1) {
ir_MERGE_WITH(end1);
}
}
ir_END_list(false_inputs);
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetExists() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (true_inputs->count) {
ir_MERGE_N(true_inputs->count, true_inputs->refs);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!op1_avoid_refcounting) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
_zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
} else if (smart_branch_opcode == ZEND_JMPNZ) {
_zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
} else {
ZEND_UNREACHABLE();
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE);
ir_END_list(end_inputs);
}
} else {
ZEND_UNREACHABLE(); // TODO: support for empty()
}
}
ir_MERGE_list(false_inputs);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!op1_avoid_refcounting) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception_undef_result(jit, opline);
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
_zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END());
} else if (smart_branch_opcode == ZEND_JMPNZ) {
_zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END());
} else {
ZEND_UNREACHABLE();
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE);
ir_END_list(end_inputs);
}
} else {
ZEND_UNREACHABLE(); // TODO: support for empty()
}
if (!exit_addr && smart_branch_opcode) {
jit->b = -1;
} else {
ir_MERGE_list(end_inputs);
}
return 1;
}
static int zend_jit_assign_dim(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint32_t val_info,
zend_jit_addr op3_addr,
zend_jit_addr op3_def_addr,
zend_jit_addr res_addr,
uint8_t dim_type,
int may_throw)
{
ir_ref if_type = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED, ht_ref;
if (op3_addr != op3_def_addr && op3_def_addr) {
if (!zend_jit_update_regs(jit, (opline+1)->op1.var, op3_addr, op3_def_addr, val_info)) {
return 0;
}
if (Z_MODE(op3_def_addr) == IS_REG && Z_MODE(op3_addr) != IS_REG) {
op3_addr = op3_def_addr;
}
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr);
val_info &= ~MAY_BE_UNDEF;
}
op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
if (opline->op2_type == IS_UNUSED) {
uint32_t var_info = MAY_BE_NULL;
ir_ref if_ok, ref;
zend_jit_addr var_addr;
// JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
ht_ref, jit_EG(uninitialized_zval));
// JIT: if (UNEXPECTED(!var_ptr)) {
if_ok = ir_IF(ref);
ir_IF_FALSE_cold(if_ok);
// JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
jit_SET_EX_OPLINE(jit, opline);
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
ir_END_list(end_inputs);
ir_IF_TRUE(if_ok);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
zend_jit_addr var_addr;
ir_ref ref;
ir_refs *found_inputs, *found_values;
ir_refs_init(found_inputs, 8);
ir_refs_init(found_values, 8);
if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info,
op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL,
0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
return 0;
}
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
// JIT: value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
if (opline->op1_type == IS_VAR
&& Z_MODE(op3_addr) != IS_REG
&& opline->result_type == IS_UNUSED
&& (res_addr == 0 || Z_MODE(res_addr) != IS_REG)) {
if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) {
return 0;
}
}
}
}
ir_END_list(end_inputs);
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
ir_ref arg2, arg4;
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IR_UNUSED;
}
jit_SET_EX_OPLINE(jit, opline);
if (opline->op2_type == IS_UNUSED) {
arg2 = IR_NULL;
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
if (opline->result_type == IS_UNUSED) {
arg4 = IR_NULL;
} else {
arg4 = jit_ZVAL_ADDR(jit, res_addr);
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, op3_addr),
arg4);
#ifdef ZEND_JIT_USE_RC_INFERENCE
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
/* ASSIGN_DIM may increase refcount of the value */
val_info |= MAY_BE_RCN;
}
#endif
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL);
ir_END_list(end_inputs);
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
ir_MERGE_list(end_inputs);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline);
if (!op1_indirect) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_assign_dim_op(zend_jit_ctx *jit,
const zend_op *opline,
uint32_t op1_info,
uint32_t op1_def_info,
zend_jit_addr op1_addr,
bool op1_indirect,
uint32_t op2_info,
zend_jit_addr op2_addr,
zend_ssa_range *op2_range,
uint32_t op1_data_info,
zend_jit_addr op3_addr,
zend_ssa_range *op1_data_range,
uint8_t dim_type,
int may_throw)
{
zend_jit_addr var_addr = IS_UNUSED;
const void *not_found_exit_addr = NULL;
uint32_t var_info = MAY_BE_NULL;
ir_ref if_type = IS_UNUSED;
ir_ref end_inputs = IR_UNUSED, ht_ref;
bool emit_fast_path = 1;
ZEND_ASSERT(opline->result_type == IS_UNUSED);
if (may_throw) {
jit_SET_EX_OPLINE(jit, opline);
}
op1_addr = zend_jit_prepare_array_update(jit, opline, op1_info, op1_addr, &if_type, &ht_ref, &may_throw);
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
if (opline->op2_type == IS_UNUSED) {
var_info = MAY_BE_NULL;
ir_ref if_ok, ref;
// JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert),
ht_ref, jit_EG(uninitialized_zval));
// JIT: if (UNEXPECTED(!var_ptr)) {
if_ok = ir_IF(ref);
ir_IF_FALSE_cold(if_ok);
// JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC));
ir_END_list(end_inputs);
ir_IF_TRUE(if_ok);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
} else {
ir_ref ref;
ir_refs *found_inputs, *found_values;
ir_refs_init(found_inputs, 8);
ir_refs_init(found_values, 8);
var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
if (dim_type != IS_UNKNOWN
&& dim_type != IS_UNDEF
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
&& (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
}
if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info,
op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, NULL,
0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) {
return 0;
}
if (found_inputs->count) {
ir_MERGE_N(found_inputs->count, found_inputs->refs);
ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
if (not_found_exit_addr && dim_type != IS_REFERENCE) {
jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr);
var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
ir_ref if_ref, if_typed, noref_path, ref_path, ref, reference, ref2, arg2;
ref = jit_ZVAL_ADDR(jit, var_addr);
if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR_ref(jit, ref);
ref2 = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
arg2 = jit_ZVAL_ADDR(jit, op3_addr);
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
reference, arg2, ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
ref = ir_PHI_2(IR_ADDR, ref, ref2);
var_addr = ZEND_ADDR_REF_ZVAL(ref);
}
} else {
emit_fast_path = 0;
}
}
if (emit_fast_path) {
uint8_t val_op_type = (opline+1)->op1_type;
if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
/* prevent FREE_OP in the helpers */
val_op_type = IS_CV;
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, may_throw)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
val_op_type, (opline+1)->op1, op3_addr, op1_data_info,
op1_data_range,
0, var_addr, var_def_info, var_info, may_throw)) {
return 0;
}
break;
case ZEND_CONCAT:
if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
may_throw)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
ir_END_list(end_inputs);
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
binary_op_type binary_op;
ir_ref arg2;
if (if_type) {
ir_IF_FALSE_cold(if_type);
if_type = IS_UNUSED;
}
if (opline->op2_type == IS_UNUSED) {
arg2 = IR_NULL;
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1);
} else {
arg2 = jit_ZVAL_ADDR(jit, op2_addr);
}
binary_op = get_binary_op(opline->extended_value);
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_op_helper),
jit_ZVAL_ADDR(jit, op1_addr),
arg2,
jit_ZVAL_ADDR(jit, op3_addr),
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
if (!op1_indirect) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
// JIT: ZVAL_COPY(res, value);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1);
} else {
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV);
}
// JIT: Z_FE_POS_P(res) = 0;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(0));
return 1;
}
static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
ir_ref ref;
if (!exit_addr) {
return 0;
}
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED));
if (op_info & MAY_BE_ARRAY_PACKED) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr));
}
return 1;
}
static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr)
{
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
ir_ref ref, ht_ref, hash_pos_ref, packed_pos_ref, hash_p_ref = IR_UNUSED, packed_p_ref = IR_UNUSED, if_packed = IR_UNUSED;
ir_ref if_def_hash = IR_UNUSED, if_def_packed = IR_UNUSED;
ir_ref exit_inputs = IR_UNUSED;
if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) {
/* empty array */
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} else {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END());
jit->b = -1;
}
return 1;
}
// JIT: array = EX_VAR(opline->op1.var);
// JIT: fe_ht = Z_ARRVAL_P(array);
ht_ref = jit_Z_PTR(jit, op1_addr);
if (op1_info & MAY_BE_PACKED_GUARD) {
if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) {
return 0;
}
}
// JIT: pos = Z_FE_POS_P(array);
hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)));
if (MAY_BE_HASH(op1_info)) {
ir_ref loop_ref, pos2_ref, p2_ref;
if (MAY_BE_PACKED(op1_info)) {
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))),
ir_CONST_U32(HASH_FLAG_PACKED));
if_packed = ir_IF(ref);
ir_IF_FALSE(if_packed);
}
// JIT: p = fe_ht->arData + pos;
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(hash_pos_ref);
} else {
ref = ir_BITCAST_A(hash_pos_ref);
}
hash_p_ref = ir_ADD_A(
ir_MUL_A(ref, ir_CONST_ADDR(sizeof(Bucket))),
ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arData))));
loop_ref = ir_LOOP_BEGIN(ir_END());
hash_pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, IR_UNUSED);
hash_p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, IR_UNUSED);
// JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
ref = ir_ULT(hash_pos_ref,
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
// JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
// JIT: ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
// JIT: pos++;
pos2_ref = ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1));
// JIT: value_type = Z_TYPE_INFO_P(value);
// JIT: if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref));
ir_IF_FALSE(if_def_hash);
} else {
ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr));
}
// JIT: p++;
p2_ref = ir_ADD_OFFSET(hash_p_ref, sizeof(Bucket));
ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
ir_PHI_SET_OP(hash_pos_ref, 2, pos2_ref);
ir_PHI_SET_OP(hash_p_ref, 2, p2_ref);
if (MAY_BE_PACKED(op1_info)) {
ir_IF_TRUE(if_packed);
}
}
if (MAY_BE_PACKED(op1_info)) {
ir_ref loop_ref, pos2_ref, p2_ref;
// JIT: p = fe_ht->arPacked + pos;
if (sizeof(void*) == 8) {
ref = ir_ZEXT_A(packed_pos_ref);
} else {
ref = ir_BITCAST_A(packed_pos_ref);
}
packed_p_ref = ir_ADD_A(
ir_MUL_A(ref, ir_CONST_ADDR(sizeof(zval))),
ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))));
loop_ref = ir_LOOP_BEGIN(ir_END());
packed_pos_ref = ir_PHI_2(IR_U32, packed_pos_ref, IR_UNUSED);
packed_p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, IR_UNUSED);
// JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
ref = ir_ULT(packed_pos_ref,
ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))));
// JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
// JIT: ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
ir_GUARD(ref, ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
} else {
ir_ref if_fit = ir_IF(ref);
ir_IF_FALSE(if_fit);
ir_END_list(exit_inputs);
ir_IF_TRUE(if_fit);
}
// JIT: pos++;
pos2_ref = ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1));
// JIT: value_type = Z_TYPE_INFO_P(value);
// JIT: if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref));
ir_IF_FALSE(if_def_packed);
} else {
ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr));
}
// JIT: p++;
p2_ref = ir_ADD_OFFSET(packed_p_ref, sizeof(zval));
ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
ir_PHI_SET_OP(packed_pos_ref, 2, pos2_ref);
ir_PHI_SET_OP(packed_p_ref, 2, p2_ref);
}
if (!exit_addr || exit_opcode == ZEND_JMP) {
zend_jit_addr val_addr;
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
uint32_t val_info;
ir_ref p_ref = IR_UNUSED, hash_path = IR_UNUSED;
if (RETURN_VALUE_USED(opline)) {
zend_jit_addr res_addr = RES_ADDR();
if (MAY_BE_HASH(op1_info)) {
ir_ref key_ref = IR_UNUSED, if_key = IR_UNUSED, key_path = IR_UNUSED;
ZEND_ASSERT(if_def_hash);
ir_IF_TRUE(if_def_hash);
// JIT: Z_FE_POS_P(array) = pos + 1;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1)));
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
key_ref = ir_LOAD_A(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, key)));
}
if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
&& (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
// JIT: if (!p->key) {
if_key = ir_IF(key_ref);
ir_IF_TRUE(if_key);
}
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
ir_ref if_interned, interned_path;
// JIT: ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
jit_set_Z_PTR(jit, res_addr, key_ref);
ref = ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(key_ref, offsetof(zend_refcounted, gc.u.type_info))),
ir_CONST_U32(IS_STR_INTERNED));
if_interned = ir_IF(ref);
ir_IF_TRUE(if_interned);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING);
interned_path = ir_END();
ir_IF_FALSE(if_interned);
jit_GC_ADDREF(jit, key_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX);
ir_MERGE_WITH(interned_path);
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
key_path = ir_END();
}
}
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
ir_IF_FALSE(if_key);
}
// JIT: ZVAL_LONG(EX_VAR(opline->result.var), p->h);
ref = ir_LOAD_L(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, h)));
jit_set_Z_LVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
ir_MERGE_WITH(key_path);
}
}
if (MAY_BE_PACKED(op1_info)) {
hash_path = ir_END();
} else {
p_ref = hash_p_ref;
}
}
if (MAY_BE_PACKED(op1_info)) {
ZEND_ASSERT(if_def_packed);
ir_IF_TRUE(if_def_packed);
// JIT: Z_FE_POS_P(array) = pos + 1;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1)));
// JIT: ZVAL_LONG(EX_VAR(opline->result.var), pos);
if (sizeof(zend_long) == 8) {
packed_pos_ref = ir_ZEXT_L(packed_pos_ref);
} else {
packed_pos_ref = ir_BITCAST_L(packed_pos_ref);
}
jit_set_Z_LVAL(jit, res_addr, packed_pos_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
if (MAY_BE_HASH(op1_info)) {
ir_MERGE_WITH(hash_path);
p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, hash_p_ref);
} else {
p_ref = packed_p_ref;
}
}
} else {
ir_ref pos_ref = IR_UNUSED;
if (if_def_hash && if_def_packed) {
ir_IF_TRUE(if_def_hash);
ir_MERGE_WITH_EMPTY_TRUE(if_def_packed);
pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, packed_pos_ref);
p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, packed_p_ref);
} else if (if_def_hash) {
ir_IF_TRUE(if_def_hash);
pos_ref = hash_pos_ref;
p_ref = hash_p_ref;
} else if (if_def_packed) {
ir_IF_TRUE(if_def_packed);
pos_ref = packed_pos_ref;
p_ref = packed_p_ref;
} else {
ZEND_UNREACHABLE();
}
// JIT: Z_FE_POS_P(array) = pos + 1;
ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)),
ir_ADD_U32(pos_ref, ir_CONST_U32(1)));
}
val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
if (val_info & MAY_BE_ARRAY) {
val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
}
if (op1_info & MAY_BE_ARRAY_OF_REF) {
val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
} else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1 | MAY_BE_RCN;
}
val_addr = ZEND_ADDR_REF_ZVAL(p_ref);
if (opline->op2_type == IS_CV) {
// JIT: zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 0, 1)) {
return 0;
}
} else {
// JIT: ZVAL_COPY(res, value);
jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1);
}
if (!exit_addr) {
zend_basic_block *bb;
ZEND_ASSERT(jit->b >= 0);
bb = &jit->ssa->cfg.blocks[jit->b];
_zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END());
ZEND_ASSERT(exit_inputs);
if (!jit->ctx.ir_base[exit_inputs].op2) {
ref = exit_inputs;
} else {
ir_MERGE_list(exit_inputs);
ref = ir_END();
}
_zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref);
jit->b = -1;
}
} else {
ZEND_ASSERT(exit_inputs);
ir_MERGE_list(exit_inputs);
}
return 1;
}
static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var)
{
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
ir_ref ref = jit_Z_PTR(jit, this_addr);
jit_set_Z_PTR(jit, var_addr, ref);
jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX);
jit_GC_ADDREF(jit, ref);
return 1;
}
static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool check_only)
{
if (!op_array->scope ||
(op_array->fn_flags & ZEND_ACC_STATIC) ||
((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_IMMUTABLE)) == ZEND_ACC_CLOSURE)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
!TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr);
if (JIT_G(current_frame)) {
TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
}
}
} else {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_object);
jit_SET_EX_OPLINE(jit, opline);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this));
ir_IF_TRUE(if_object);
}
}
if (!check_only) {
if (!zend_jit_load_this(jit, opline->result.var)) {
return 0;
}
}
return 1;
}
static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))), ir_CONST_ADDR(ce)),
ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_fetch_obj(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
bool op1_avoid_refcounting,
zend_class_entry *trace_ce,
zend_jit_addr res_addr,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_property_info *prop_info;
bool may_be_dynamic = 1;
zend_jit_addr prop_addr;
uint32_t res_info = RES_INFO();
ir_ref prop_type_ref = IR_UNUSED;
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
ir_ref end_values = IR_UNUSED;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
ir_ref op1_ref = IR_UNUSED;
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
ir_ref fast_path = IR_UNUSED;
if (op1_info & MAY_BE_ANY) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
fast_path = ir_END();
ir_IF_FALSE_cold(if_def);
}
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
ir_CONST_U32(opline->op1.var));
if (fast_path) {
ir_MERGE_WITH(fast_path);
}
op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr);
} else {
op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
}
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_write),
op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
} else {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_read),
op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member)));
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
} else {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
ir_END_list(end_inputs);
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref,
ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array);
if (may_be_dynamic) {
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
} else {
ir_IF_TRUE_cold(if_dynamic);
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
if (Z_MODE(res_addr) == IS_REG) {
ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic_ex),
obj_ref, offset_ref);
ir_END_PHI_list(end_values, val_addr);
} else {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic),
obj_ref, offset_ref);
ir_END_list(end_inputs);
}
} else {
if (Z_MODE(res_addr) == IS_REG) {
ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic_ex),
obj_ref, offset_ref);
ir_END_PHI_list(end_values, val_addr);
} else {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic),
obj_ref, offset_ref);
ir_END_list(end_inputs);
}
}
}
ir_IF_FALSE(if_dynamic);
}
prop_ref = ir_ADD_A(obj_ref, offset_ref);
prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref);
ir_ref if_def = ir_IF(prop_type_ref);
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
ir_ref allowed_inputs = IR_UNUSED;
ir_ref forbidden_inputs = IR_UNUSED;
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
ir_ref prop_flags = ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, flags)));
ir_ref if_readonly_or_avis = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY|ZEND_ACC_PPP_SET_MASK)));
ir_IF_FALSE(if_readonly_or_avis);
ir_END_list(allowed_inputs);
ir_IF_TRUE_cold(if_readonly_or_avis);
ir_ref if_readonly = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY)));
ir_IF_TRUE(if_readonly);
ir_END_list(forbidden_inputs);
ir_IF_FALSE(if_readonly);
ir_ref has_avis_access = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_asymmetric_property_has_set_access), prop_info_ref);
ir_ref if_avis_access = ir_IF(has_avis_access);
ir_IF_TRUE(if_avis_access);
ir_END_list(allowed_inputs);
ir_IF_FALSE(if_avis_access);
ir_END_list(forbidden_inputs);
ir_MERGE_list(forbidden_inputs);
ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
ir_IF_TRUE(if_prop_obj);
ref = jit_Z_PTR(jit, prop_addr);
jit_GC_ADDREF(jit, ref);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
ir_END_list(end_inputs);
ir_IF_FALSE_cold(if_prop_obj);
jit_SET_EX_OPLINE(jit, opline);
if_readonly = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY)));
ir_IF_TRUE(if_readonly);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_indirect_modification_error), prop_info_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
ir_END_list(end_inputs);
ir_IF_FALSE(if_readonly);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_asymmetric_visibility_property_modification_error),
prop_info_ref, ir_CONST_ADDR("indirectly modify"));
jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
ir_END_list(end_inputs);
ir_MERGE_list(allowed_inputs);
if (flags == ZEND_FETCH_DIM_WRITE) {
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
prop_ref, prop_info_ref);
ir_END_list(end_inputs);
ir_IF_FALSE(if_has_prop_info);
} else if (flags == ZEND_FETCH_REF) {
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
prop_ref,
prop_info_ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
ir_IF_FALSE(if_has_prop_info);
} else {
ZEND_ASSERT(flags == 0);
ir_MERGE_WITH_EMPTY_FALSE(if_has_prop_info);
}
}
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
ir_GUARD(prop_type_ref, ir_CONST_ADDR(exit_addr));
}
} else {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
ir_ref if_def = ir_IF(prop_type_ref);
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
ir_IF_TRUE(if_prop_obj);
ir_ref ref = jit_Z_PTR(jit, prop_addr);
jit_GC_ADDREF(jit, ref);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
ir_END_list(end_inputs);
ir_IF_FALSE_cold(if_prop_obj);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_indirect_modification_error), ir_CONST_ADDR(prop_info));
jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR);
ir_END_list(end_inputs);
goto result_fetched;
} else if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_PPP_SET_MASK)) {
/* Readonly properties which are also asymmetric are never mutable indirectly, which is
* handled by the previous branch. */
ir_ref has_access = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_asymmetric_property_has_set_access), ir_CONST_ADDR(prop_info));
ir_ref if_access = ir_IF(has_access);
ir_IF_FALSE_cold(if_access);
ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT);
ir_IF_TRUE(if_prop_obj);
ir_ref ref = jit_Z_PTR(jit, prop_addr);
jit_GC_ADDREF(jit, ref);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX);
ir_END_list(end_inputs);
ir_IF_FALSE_cold(if_prop_obj);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_asymmetric_visibility_property_modification_error),
ir_CONST_ADDR(prop_info), ir_CONST_ADDR("indirectly modify"));
ir_END_list(end_inputs);
ir_IF_TRUE(if_access);
}
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
&& ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
if (flags == ZEND_FETCH_DIM_WRITE) {
if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) {
if (!prop_type_ref) {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
}
ir_ref if_null_or_flase = ir_IF(ir_LE(prop_type_ref, ir_CONST_U32(IR_FALSE)));
ir_IF_TRUE_cold(if_null_or_flase);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion),
prop_ref, ir_CONST_ADDR(prop_info));
ir_END_list(end_inputs);
ir_IF_FALSE(if_null_or_flase);
}
} else if (flags == ZEND_FETCH_REF) {
ir_ref ref;
if (!prop_type_ref) {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
}
ir_ref if_reference = ir_IF(ir_EQ(prop_type_ref, ir_CONST_U32(IS_REFERENCE_EX)));
ir_IF_FALSE(if_reference);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref),
prop_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
ir_IF_TRUE(if_reference);
} else {
ZEND_UNREACHABLE();
}
}
}
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ZEND_ASSERT(prop_ref);
jit_set_Z_PTR(jit, res_addr, prop_ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
ssa->var_info[ssa_op->result_def].indirect_reference = 1;
}
ir_END_list(end_inputs);
} else {
if (((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info)
|| Z_MODE(res_addr) == IS_REG) {
ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr));
} else {
prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr);
if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) {
return 0;
}
ir_END_list(end_inputs);
}
}
result_fetched:
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
ir_MERGE_list(slow_inputs);
jit_SET_EX_OPLINE(jit, opline);
if (opline->opcode == ZEND_FETCH_OBJ_W) {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_w_slow), obj_ref);
ir_END_list(end_inputs);
} else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
if (Z_MODE(res_addr) == IS_REG) {
ir_ref val_ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow_ex), obj_ref);
ir_END_PHI_list(end_values, val_ref);
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow), obj_ref);
ir_END_list(end_inputs);
}
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_slow), obj_ref);
ir_END_list(end_inputs);
}
}
if (end_values) {
ir_ref val_ref = ir_PHI_list(end_values);
zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val_ref);
bool result_avoid_refcounting = 0;
ZEND_ASSERT(opline->opcode == ZEND_FETCH_OBJ_R
|| opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG
|| opline->opcode == ZEND_FETCH_OBJ_IS);
ZEND_ASSERT(end_inputs == IR_UNUSED);
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
uint8_t type = concrete_type(res_info);
uint32_t flags = 0;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !delayed_fetch_this
&& !op1_avoid_refcounting) {
flags = ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type,
1, flags, op1_avoid_refcounting);
if (!val_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
}
// ZVAL_COPY
jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting);
if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) {
return 0;
}
} else {
ir_MERGE_list(end_inputs);
}
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_RC1)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
ir_ref if_refcounted, ptr, refcount, if_non_zero;
ir_ref merge_inputs = IR_UNUSED;
if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr);
ir_IF_FALSE( if_refcounted);
ir_END_list(merge_inputs);
ir_IF_TRUE( if_refcounted);
ptr = jit_Z_PTR(jit, orig_op1_addr);
refcount = jit_GC_DELREF(jit, ptr);
if_non_zero = ir_IF(refcount);
ir_IF_TRUE( if_non_zero);
ir_END_list(merge_inputs);
ir_IF_FALSE( if_non_zero);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_extract_helper), ptr);
ir_END_list(merge_inputs);
ir_MERGE_list(merge_inputs);
} else if (!op1_avoid_refcounting) {
if (on_this) {
op1_info &= ~MAY_BE_RC1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& prop_info
&& (opline->opcode != ZEND_FETCH_OBJ_W ||
!(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
!ZEND_TYPE_IS_SET(prop_info->type))
&& (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
may_throw = 0;
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_assign_obj(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_jit_addr val_addr,
zend_jit_addr val_def_addr,
zend_jit_addr res_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr prop_addr;
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref delayed_end_input = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
uint32_t res_info = RES_INFO();
if (val_addr != val_def_addr && val_def_addr) {
if (!zend_jit_update_regs(jit, (opline+1)->op1.var, val_addr, val_def_addr, val_info)) {
return 0;
}
if (Z_MODE(val_def_addr) == IS_REG && Z_MODE(val_addr) != IS_REG) {
val_addr = val_def_addr;
}
}
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
jit_ZVAL_ADDR(jit, op1_addr),
ir_CONST_ADDR(ZSTR_VAL(name)));
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) != IS_REG) {
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
}
ir_END_list(end_inputs);
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_dynamic);
prop_ref = ir_ADD_A(obj_ref, offset_ref);
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
ir_ref arg3, arg4;
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!RETURN_VALUE_USED(opline)) {
arg4 = IR_NULL;
} else if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
arg4 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg4 = jit_ZVAL_ADDR(jit, res_addr);
}
// JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
prop_ref,
prop_info_ref,
arg3,
arg4);
if ((opline+1)->op1_type == IS_CONST) {
// TODO: ???
// if (Z_TYPE_P(value) == orig_type) {
// CACHE_PTR_EX(cache_slot + 2, NULL);
}
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_END_list(end_inputs);
ir_IF_FALSE(if_has_prop_info);
}
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
/* With the exception of __clone(), readonly assignment always happens on IS_UNDEF, doding
* the fast path. Thus, the fast path is not useful. */
if (prop_info->flags & ZEND_ACC_READONLY) {
ZEND_ASSERT(slow_inputs == IR_UNUSED);
goto slow_path;
}
// Undefined property with potential magic __get()/__set() or lazy object
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
ir_ref ref, arg3, arg4;
// JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
jit_SET_EX_OPLINE(jit, opline);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!RETURN_VALUE_USED(opline)) {
arg4 = IR_NULL;
} else if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
arg4 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg4 = jit_ZVAL_ADDR(jit, res_addr);
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop),
prop_ref,
ref,
arg3,
arg4);
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_END_list(end_inputs);
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
if (Z_MODE(val_addr) != IS_REG
&& (res_addr == 0 || Z_MODE(res_addr) != IS_REG)
&& opline->result_type == IS_UNUSED) {
if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0, 0)) {
return 0;
}
}
if (end_inputs || slow_inputs) {
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
/* skip FREE_OP_DATA() */
delayed_end_input = ir_END();
} else {
ir_END_list(end_inputs);
}
}
}
if (slow_inputs) {
ir_ref arg3, arg5;
ir_MERGE_list(slow_inputs);
slow_path:
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
if (!RETURN_VALUE_USED(opline)) {
arg5 = IR_NULL;
} else if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
arg5 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg5 = jit_ZVAL_ADDR(jit, res_addr);
}
// JIT: value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_helper),
obj_ref,
ir_CONST_ADDR(name),
arg3,
ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
arg5);
if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) {
return 0;
}
}
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
if (delayed_end_input) {
ir_MERGE_WITH(delayed_end_input);
}
}
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_assign_obj_op(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_jit_addr val_addr,
zend_ssa_range *val_range,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr prop_addr;
bool use_prop_guard = 0;
bool may_throw = 0;
binary_op_type binary_op = get_binary_op(opline->extended_value);
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
ZEND_ASSERT(opline->result_type == IS_UNUSED);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID,
(op1_info & MAY_BE_UNDEF) ?
ir_CONST_FC_FUNC(zend_jit_invalid_property_assign_op) :
ir_CONST_FC_FUNC(zend_jit_invalid_property_assign),
jit_ZVAL_ADDR(jit, op1_addr),
ir_CONST_ADDR(ZSTR_VAL(name)));
may_throw = 1;
ir_END_list(end_inputs);
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_has_prop_info);
}
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_dynamic);
prop_ref = ir_ADD_A(obj_ref, offset_ref);
if (!use_prop_guard) {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, val_addr);
}
if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR(jit, prop_addr);
ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
reference,
arg2,
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
// JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_prop),
prop_ref,
ref,
arg2,
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
zend_jit_addr var_addr = prop_addr;
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2;
may_throw = 1;
if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR(jit, var_addr);
ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
jit_SET_EX_OPLINE(jit, opline);
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg2 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg2 = jit_ZVAL_ADDR(jit, val_addr);
}
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref),
reference,
arg2,
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
var_info &= ~MAY_BE_REF;
}
uint8_t val_op_type = (opline+1)->op1_type;
if (val_op_type & (IS_TMP_VAR|IS_VAR)) {
/* prevent FREE_OP in the helpers */
val_op_type = IS_CV;
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, 0)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
(val_info & MAY_BE_ANY) != MAY_BE_STRING) {
may_throw = 1;
}
}
goto long_math;
case ZEND_SL:
case ZEND_SR:
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
}
if (val_op_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
may_throw = 1;
}
goto long_math;
case ZEND_MOD:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
}
if (val_op_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
may_throw = 1;
}
long_math:
if (!zend_jit_long_math_helper(jit, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
val_op_type, (opline+1)->op1, val_addr, val_info,
val_range,
0, var_addr, var_def_info, var_info, /* may throw */ 1)) {
return 0;
}
break;
case ZEND_CONCAT:
may_throw = 1;
if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, var_addr,
0)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
if (end_inputs || slow_inputs) {
ir_END_list(end_inputs);
}
}
if (slow_inputs) {
ir_ref arg3;
ir_MERGE_list(slow_inputs);
may_throw = 1;
if (Z_MODE(val_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var);
if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) {
return 0;
}
arg3 = jit_ZVAL_ADDR(jit, real_addr);
} else {
arg3 = jit_ZVAL_ADDR(jit, val_addr);
}
jit_SET_EX_OPLINE(jit, opline);
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_op_helper),
obj_ref,
ir_CONST_ADDR(name),
arg3,
ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
ir_CONST_FC_FUNC(binary_op));
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
// JIT: FREE_OP_DATA();
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline);
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_incdec_obj(zend_jit_ctx *jit,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr res_addr = 0;
zend_jit_addr prop_addr;
bool use_prop_guard = 0;
bool may_throw = 0;
uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;
ir_ref obj_ref = IR_UNUSED;
ir_ref prop_ref = IR_UNUSED;
ir_ref end_inputs = IR_UNUSED;
ir_ref slow_inputs = IR_UNUSED;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
if (opline->result_type != IS_UNUSED) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
obj_ref = jit_Z_PTR(jit, this_addr);
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr);
} else {
ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT);
ir_IF_FALSE_cold(if_obj);
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_incdec),
jit_ZVAL_ADDR(jit, op1_addr),
ir_CONST_ADDR(ZSTR_VAL(name)));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler));
ir_IF_TRUE(if_obj);
}
}
obj_ref = jit_Z_PTR(jit, op1_addr);
}
ZEND_ASSERT(obj_ref);
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS));
ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)))));
ir_IF_FALSE_cold(if_same);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_same);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
ir_ref prop_info_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2));
ir_ref if_has_prop_info = ir_IF(prop_info_ref);
ir_IF_TRUE_cold(if_has_prop_info);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_has_prop_info);
}
ir_ref offset_ref = ir_LOAD_A(
ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)));
ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET)));
ir_IF_TRUE_cold(if_dynamic);
ir_END_list(slow_inputs);
ir_IF_FALSE(if_dynamic);
prop_ref = ir_ADD_A(obj_ref, offset_ref);
if (!use_prop_guard) {
ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
} else {
prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset);
prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr));
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr));
ir_IF_FALSE_cold(if_def);
ir_END_list(slow_inputs);
ir_IF_TRUE(if_def);
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
const void *func;
ir_ref ref;
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
ref = ir_CONST_ADDR(prop_info);
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table)));
ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset));
}
if (opline->result_type == IS_UNUSED) {
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
case ZEND_POST_INC_OBJ:
func = zend_jit_inc_typed_prop;
break;
case ZEND_PRE_DEC_OBJ:
case ZEND_POST_DEC_OBJ:
func = zend_jit_dec_typed_prop;
break;
default:
ZEND_UNREACHABLE();
}
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func), prop_ref, ref);
} else {
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
func = zend_jit_pre_inc_typed_prop;
break;
case ZEND_PRE_DEC_OBJ:
func = zend_jit_pre_dec_typed_prop;
break;
case ZEND_POST_INC_OBJ:
func = zend_jit_post_inc_typed_prop;
break;
case ZEND_POST_DEC_OBJ:
func = zend_jit_post_dec_typed_prop;
break;
default:
ZEND_UNREACHABLE();
}
ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(func),
prop_ref,
ref,
jit_ZVAL_ADDR(jit, res_addr));
}
ir_END_list(end_inputs);
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
zend_jit_addr var_addr = prop_addr;
ir_ref if_long = IR_UNUSED;
ir_ref if_overflow = IR_UNUSED;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr);
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
const void *func;
ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref;
if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE);
ir_IF_FALSE(if_ref);
noref_path = ir_END();
ir_IF_TRUE(if_ref);
reference = jit_Z_PTR(jit, var_addr);
ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val));
if_typed = jit_if_TYPED_REF(jit, reference);
ir_IF_FALSE(if_typed);
ref_path = ir_END();
ir_IF_TRUE_cold(if_typed);
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
func = zend_jit_pre_inc_typed_ref;
break;
case ZEND_PRE_DEC_OBJ:
func = zend_jit_pre_dec_typed_ref;
break;
case ZEND_POST_INC_OBJ:
func = zend_jit_post_inc_typed_ref;
break;
case ZEND_POST_DEC_OBJ:
func = zend_jit_post_dec_typed_ref;
break;
default:
ZEND_UNREACHABLE();
}
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func),
reference,
(opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
ir_MERGE_2(noref_path, ref_path);
prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref);
var_addr = ZEND_ADDR_REF_ZVAL(prop_ref);
var_info &= ~MAY_BE_REF;
}
if (var_info & MAY_BE_LONG) {
ir_ref addr, ref;
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG);
ir_IF_TRUE(if_long);
}
addr = jit_ZVAL_ADDR(jit, var_addr);
ref = ir_LOAD_L(addr);
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
ref = ir_ADD_OV_L(ref, ir_CONST_LONG(1));
} else {
ref = ir_SUB_OV_L(ref, ir_CONST_LONG(1));
}
ir_STORE(addr, ref);
if_overflow = ir_IF(ir_OVERFLOW(ref));
ir_IF_FALSE(if_overflow);
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
jit_set_Z_LVAL(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG);
}
}
ir_END_list(end_inputs);
}
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
may_throw = 1;
}
if (if_long) {
ir_IF_FALSE_cold(if_long);
}
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1);
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc),
jit_ZVAL_ADDR(jit, var_addr),
jit_ZVAL_ADDR(jit, res_addr));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function),
jit_ZVAL_ADDR(jit, var_addr));
}
} else {
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec),
jit_ZVAL_ADDR(jit, var_addr),
jit_ZVAL_ADDR(jit, res_addr));
} else {
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function),
jit_ZVAL_ADDR(jit, var_addr));
}
}
ir_END_list(end_inputs);
}
if (var_info & MAY_BE_LONG) {
ir_IF_TRUE_cold(if_overflow);
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
} else {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE);
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
#if SIZEOF_ZEND_LONG == 4
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000));
jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000));
#else
jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000));
#endif
jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE);
}
}
if (opline->result_type != IS_UNUSED
&& (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
&& prop_info
&& !ZEND_TYPE_IS_SET(prop_info->type)
&& (res_info & MAY_BE_GUARD)
&& (res_info & MAY_BE_LONG)) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
int32_t exit_point;
const void *exit_addr;
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
} else {
ir_END_list(end_inputs);
}
}
}
if (slow_inputs) {
const void *func;
ir_MERGE_list(slow_inputs);
// JIT: zend_jit_pre_inc_obj_helper(zobj, name, CACHE_ADDR(opline->extended_value), result);
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
func = zend_jit_pre_inc_obj_helper;
break;
case ZEND_PRE_DEC_OBJ:
func = zend_jit_pre_dec_obj_helper;
break;
case ZEND_POST_INC_OBJ:
func = zend_jit_post_inc_obj_helper;
break;
case ZEND_POST_DEC_OBJ:
func = zend_jit_post_dec_obj_helper;
break;
default:
ZEND_UNREACHABLE();
}
may_throw = 1;
jit_SET_EX_OPLINE(jit, opline);
ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache));
ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(func),
obj_ref,
ir_CONST_ADDR(name),
ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS),
(opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr));
ir_END_list(end_inputs);
}
if (end_inputs) {
ir_MERGE_list(end_inputs);
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline);
}
if (may_throw) {
zend_jit_check_exception(jit);
}
return 1;
}
static int zend_jit_fetch_static_prop(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array)
{
zend_jit_addr res_addr = RES_ADDR();
uint32_t cache_slot = opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS;
uint32_t flags;
ir_ref ref, ref2, if_cached, fast_path, cold_path, prop_info_ref, if_typed, if_def;
int fetch_type;
zend_property_info *known_prop_info = NULL;
zend_class_entry *ce;
ce = zend_get_known_class(op_array, opline, opline->op2_type, opline->op2);
if (ce) {
zval *zv = RT_CONSTANT(opline, opline->op1);
zend_string *prop_name;
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
prop_name = Z_STR_P(zv);
zv = zend_hash_find(&ce->properties_info, prop_name);
if (zv) {
zend_property_info *prop_info = Z_PTR_P(zv);
if (prop_info->flags & ZEND_ACC_STATIC) {
if (prop_info->ce == op_array->scope
|| (prop_info->flags & ZEND_ACC_PUBLIC)
|| ((prop_info->flags & ZEND_ACC_PROTECTED)
&& op_array->scope
&& instanceof_function_slow(op_array->scope, prop_info->ce))) {
known_prop_info = prop_info;
}
}
}
}
switch (opline->opcode) {
case ZEND_FETCH_STATIC_PROP_R:
case ZEND_FETCH_STATIC_PROP_FUNC_ARG:
fetch_type = BP_VAR_R;
break;
case ZEND_FETCH_STATIC_PROP_IS:
fetch_type = BP_VAR_IS;
break;
case ZEND_FETCH_STATIC_PROP_W:
fetch_type = BP_VAR_W;
break;
case ZEND_FETCH_STATIC_PROP_RW:
fetch_type = BP_VAR_RW;
break;
case ZEND_FETCH_STATIC_PROP_UNSET:
fetch_type = BP_VAR_UNSET;
break;
EMPTY_SWITCH_DEFAULT_CASE();
}
// JIT: result = CACHED_PTR(cache_slot + sizeof(void *));
ref = ir_LOAD_A(
ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), cache_slot + sizeof(void*)));
// JIT: if (result)
if_cached = ir_IF(ref);
ir_IF_TRUE(if_cached);
if (fetch_type == BP_VAR_R || fetch_type == BP_VAR_RW) {
if (!known_prop_info || ZEND_TYPE_IS_SET(known_prop_info->type)) {
ir_ref merge = IR_UNUSED;
// JIT: if (UNEXPECTED(Z_TYPE_P(result) == IS_UNDEF)
if_typed = IR_UNUSED;
if_def = ir_IF(jit_Z_TYPE_ref(jit, ref));
ir_IF_FALSE_cold(if_def);
if (!known_prop_info) {
// JIT: if (ZEND_TYPE_IS_SET(property_info->type))
prop_info_ref = ir_LOAD_L(
ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), cache_slot + sizeof(void*) * 2));
if_typed = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, type.type_mask))),
ir_CONST_U32(_ZEND_TYPE_MASK)));
ir_IF_FALSE(if_typed);
ir_END_list(merge);
ir_IF_TRUE(if_typed);
}
// JIT: zend_throw_error(NULL, "Typed static property %s::$%s must not be accessed before initialization",
// ZSTR_VAL(property_info->ce->name),
// zend_get_unmangled_property_name(property_info->name));
jit_SET_EX_OPLINE(jit, opline);
ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_uninit_static_prop));
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
ir_IF_TRUE(if_def);
if (!known_prop_info) {
ir_END_list(merge);
ir_MERGE_list(merge);
}
}
} else if (fetch_type == BP_VAR_W) {
flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
if (flags && (!known_prop_info || ZEND_TYPE_IS_SET(known_prop_info->type))) {
ir_ref merge = IR_UNUSED;
if (!known_prop_info) {
// JIT: if (ZEND_TYPE_IS_SET(property_info->type))
prop_info_ref = ir_LOAD_L(
ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), cache_slot + sizeof(void*) * 2));
if_typed = ir_IF(ir_AND_U32(
ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, type.type_mask))),
ir_CONST_U32(_ZEND_TYPE_MASK)));
ir_IF_FALSE(if_typed);
ir_END_list(merge);
ir_IF_TRUE(if_typed);
} else {
prop_info_ref = ir_CONST_ADDR(known_prop_info);
}
// JIT: zend_handle_fetch_obj_flags(NULL, *retval, NULL, property_info, flags);
ir_ref if_ok = ir_IF(ir_CALL_5(IR_BOOL, ir_CONST_FUNC(zend_handle_fetch_obj_flags),
IR_NULL, ref, IR_NULL, prop_info_ref, ir_CONST_U32(flags)));
ir_IF_FALSE_cold(if_ok);
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef));
ir_IF_TRUE(if_ok);
if (!known_prop_info) {
ir_END_list(merge);
ir_MERGE_list(merge);
}
}
}
fast_path = ir_END();
ir_IF_FALSE_cold(if_cached);
jit_SET_EX_OPLINE(jit, opline);
ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_fetch_static_property), jit_FP(jit), ir_CONST_I32(fetch_type));
zend_jit_check_exception_undef_result(jit, opline);
cold_path = ir_END();
ir_MERGE_2(fast_path, cold_path);
ref = ir_PHI_2(IR_ADDR, ref, ref2);
if (fetch_type == BP_VAR_R || fetch_type == BP_VAR_IS) {
// JIT: ZVAL_COPY_DEREF(EX_VAR(opline->result.var), result);
if (!zend_jit_zval_copy_deref(jit, res_addr, ZEND_ADDR_REF_ZVAL(ref),
jit_Z_TYPE_INFO_ref(jit, ref))) {
return 0;
}
} else {
// JIT: ZVAL_INDIRECT(EX_VAR(opline->result.var), result);
jit_set_Z_PTR(jit, res_addr, ref);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT);
}
return 1;
}
static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
{
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *next_opline = NULL;
ir_refs *slow_inputs;
ir_refs_init(slow_inputs, 8);
if (trace) {
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
ZEND_ASSERT(trace->opline != NULL);
next_opline = trace->opline;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
zval *jump_zv = NULL;
int b;
if (opline->opcode == ZEND_SWITCH_LONG) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else if (opline->opcode == ZEND_MATCH) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
} else if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else {
ZEND_UNREACHABLE();
}
if (next_opline) {
const zend_op *target;
if (jump_zv != NULL) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
}
ZEND_ASSERT(target == next_opline);
} else {
if (jump_zv != NULL) {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
} else {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
}
_zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END());
jit->b = -1;
}
} else {
zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL;
uint32_t op1_info = OP1_INFO();
zend_jit_addr op1_addr = OP1_ADDR();
const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
const zend_op *target;
int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
int b;
int32_t exit_point;
const void *exit_addr;
const void *default_label = NULL;
zval *zv;
if (next_opline) {
if (next_opline != default_opline) {
exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
default_label = zend_jit_trace_get_exit_addr(exit_point);
if (!default_label) {
return 0;
}
}
}
if (opline->opcode == ZEND_SWITCH_LONG) {
if (op1_info & MAY_BE_LONG) {
const void *fallback_label = NULL;
if (next_opline) {
if (next_opline != opline + 1) {
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
fallback_label = zend_jit_trace_get_exit_addr(exit_point);
if (!fallback_label) {
return 0;
}
}
}
if (op1_info & MAY_BE_REF) {
ir_ref ref, if_long, fast_path, ref2;
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_long);
fast_path = ir_END();
ir_IF_FALSE_cold(if_long);
// JIT: ZVAL_DEREF(op)
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
} else {
ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE);
ir_IF_FALSE_cold(if_ref);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_ref);
}
ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
} else {
if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_long);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_long);
}
ir_MERGE_2(fast_path, ir_END());
ref = ir_PHI_2(IR_ADDR, ref, ref2);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
} else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
} else {
ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE_cold(if_long);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_long);
}
}
ir_ref ref = jit_Z_LVAL(jit, op1_addr);
if (!HT_IS_PACKED(jumptable)) {
ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
ir_CONST_ADDR(jumptable), ref);
ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
/* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
if (sizeof(Bucket) == 32) {
ref = ir_SHR_L(ref, ir_CONST_LONG(5));
} else {
ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
}
}
ref = ir_SWITCH(ref);
if (next_opline) {
ir_ref continue_list = IR_UNUSED;
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
ir_ref idx;
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(ref, idx);
if (target == next_opline) {
ir_END_list(continue_list);
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} ZEND_HASH_FOREACH_END();
ir_CASE_DEFAULT(ref);
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
if (continue_list) {
ir_MERGE_list(continue_list);
} else {
ZEND_ASSERT(slow_inputs->count);
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
}
} else {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
b = ssa->cfg.map[target - op_array->opcodes];
_zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
} ZEND_HASH_FOREACH_END();
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
if (slow_inputs->count) {
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
}
jit->b = -1;
}
} else if (!next_opline) {
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
jit->b = -1;
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (op1_info & MAY_BE_STRING) {
const void *fallback_label = NULL;
if (next_opline) {
if (next_opline != opline + 1) {
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
fallback_label = zend_jit_trace_get_exit_addr(exit_point);
if (!fallback_label) {
return 0;
}
}
}
if (op1_info & MAY_BE_REF) {
ir_ref ref, if_string, fast_path, ref2;
ref = jit_ZVAL_ADDR(jit, op1_addr);
if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_string);
fast_path = ir_END();
ir_IF_FALSE_cold(if_string);
// JIT: ZVAL_DEREF(op)
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label);
} else {
ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE_cold(if_ref);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_ref);
}
ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val));
op1_addr = ZEND_ADDR_REF_ZVAL(ref2);
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label);
} else {
if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE_cold(if_string);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_string);
}
ir_MERGE_2(fast_path, ir_END());
ref = ir_PHI_2(IR_ADDR, ref, ref2);
op1_addr = ZEND_ADDR_REF_ZVAL(ref);
} else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
if (fallback_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label);
} else {
ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE_cold(if_string);
ir_refs_add(slow_inputs, ir_END());
ir_IF_TRUE(if_string);
}
}
ir_ref ref = jit_Z_PTR(jit, op1_addr);
ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
ir_CONST_ADDR(jumptable), ref);
ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
/* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
if (sizeof(Bucket) == 32) {
ref = ir_SHR_L(ref, ir_CONST_LONG(5));
} else {
ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
}
ref = ir_SWITCH(ref);
if (next_opline) {
ir_ref continue_list = IR_UNUSED;
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
ir_ref idx;
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(ref, idx);
if (target == next_opline) {
ir_END_list(continue_list);
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} ZEND_HASH_FOREACH_END();
ir_CASE_DEFAULT(ref);
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
if (continue_list) {
ir_MERGE_list(continue_list);
} else {
ZEND_ASSERT(slow_inputs->count);
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
}
} else {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
b = ssa->cfg.map[target - op_array->opcodes];
_zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
} ZEND_HASH_FOREACH_END();
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
if (slow_inputs->count) {
ir_MERGE_N(slow_inputs->count, slow_inputs->refs);
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
}
jit->b = -1;
}
} else if (!next_opline) {
_zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END());
jit->b = -1;
}
} else if (opline->opcode == ZEND_MATCH) {
ir_ref if_type = IR_UNUSED, default_input_list = IR_UNUSED, ref = IR_UNUSED;
ir_ref continue_list = IR_UNUSED;
if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
ir_ref long_path = IR_UNUSED;
if (op1_info & MAY_BE_REF) {
op1_addr = jit_ZVAL_DEREF(jit, op1_addr);
}
if (op1_info & MAY_BE_LONG) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & (MAY_BE_STRING|MAY_BE_UNDEF)) {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_TRUE(if_type);
} else if (default_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label);
} else if (next_opline) {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE(if_type);
ir_END_list(continue_list);
ir_IF_TRUE(if_type);
} else {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG);
ir_IF_FALSE(if_type);
ir_END_list(default_input_list);
ir_IF_TRUE(if_type);
}
}
ref = jit_Z_LVAL(jit, op1_addr);
ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find),
ir_CONST_ADDR(jumptable), ref);
if (op1_info & MAY_BE_STRING) {
long_path = ir_END();
}
}
if (op1_info & MAY_BE_STRING) {
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IS_UNUSED;
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
if (op1_info & MAY_BE_UNDEF) {
if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_TRUE(if_type);
} else if (default_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label);
} else if (next_opline) {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE(if_type);
ir_END_list(continue_list);
ir_IF_TRUE(if_type);
} else {
ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING);
ir_IF_FALSE(if_type);
ir_END_list(default_input_list);
ir_IF_TRUE(if_type);
}
}
ir_ref ref2 = jit_Z_PTR(jit, op1_addr);
ref2 = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find),
ir_CONST_ADDR(jumptable), ref2);
if (op1_info & MAY_BE_LONG) {
ir_MERGE_WITH(long_path);
ref = ir_PHI_2(IR_LONG, ref2, ref);
} else {
ref = ref2;
}
}
ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData));
/* Signed DIV by power of 2 may be optimized into SHR only for positive operands */
if (HT_IS_PACKED(jumptable)) {
ZEND_ASSERT(sizeof(zval) == 16);
ref = ir_SHR_L(ref, ir_CONST_LONG(4));
} else {
if (sizeof(Bucket) == 32) {
ref = ir_SHR_L(ref, ir_CONST_LONG(5));
} else {
ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket)));
}
}
ref = ir_SWITCH(ref);
if (next_opline) {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
ir_ref idx;
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (HT_IS_PACKED(jumptable)) {
idx = ir_CONST_LONG(zv - jumptable->arPacked);
} else {
idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData);
}
ir_CASE_VAL(ref, idx);
if (target == next_opline) {
ir_END_list(continue_list);
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr));
}
} ZEND_HASH_FOREACH_END();
ir_CASE_DEFAULT(ref);
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
} else {
ZEND_HASH_FOREACH_VAL(jumptable, zv) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
b = ssa->cfg.map[target - op_array->opcodes];
_zend_jit_add_predecessor_ref(jit, b, jit->b, ref);
} ZEND_HASH_FOREACH_END();
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref);
}
} else if (!(op1_info & MAY_BE_UNDEF)) {
if (next_opline) {
if (next_opline == default_opline) {
ir_END_list(continue_list);
} else {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
}
} else {
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
}
}
if (op1_info & MAY_BE_UNDEF) {
if (if_type) {
ir_IF_FALSE(if_type);
if_type = IS_UNUSED;
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
if (default_label) {
jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label);
} else if (next_opline) {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
ir_END_list(continue_list);
ir_IF_FALSE_cold(if_def);
} else {
ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr));
ir_IF_TRUE(if_def);
ir_END_list(default_input_list);
ir_IF_FALSE_cold(if_def);
}
}
jit_SET_EX_OPLINE(jit, opline);
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper),
ir_CONST_U32(opline->op1.var));
zend_jit_check_exception_undef_result(jit, opline);
if (default_label) {
jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label));
} else if (next_opline) {
ir_END_list(continue_list);
} else {
ir_END_list(default_input_list);
}
}
if (next_opline) {
ZEND_ASSERT(continue_list);
ir_MERGE_list(continue_list);
} else {
if (default_input_list) {
if (jit->ctx.ir_base[ref].op == IR_SWITCH) {
ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED);
jit->ctx.ir_base[ref].op3 = default_input_list;
} else {
ir_MERGE_list(default_input_list);
_zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END());
}
}
jit->b = -1;
}
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa)
{
int i, count;
zend_basic_block *bb;
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET));
jit->ctx.spill_base = ZREG_FP;
jit->op_array = jit->current_op_array = op_array;
jit->ssa = ssa;
jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref));
jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count;
count = 0;
for (i = 0, bb = ssa->cfg.blocks; i < ssa->cfg.blocks_count; i++, bb++) {
jit->bb_predecessors[i] = count;
count += bb->predecessors_count;
}
jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref));
if (!GCC_GLOBAL_REGS) {
ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
jit_STORE_FP(jit, ref);
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
return 1;
}
static void *zend_jit_finish(zend_jit_ctx *jit)
{
void *entry;
size_t size;
zend_string *str = NULL;
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP|
ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_SCCP|
ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS|ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) {
if (jit->name) {
str = zend_string_copy(jit->name);
} else {
str = zend_jit_func_name(jit->op_array);
}
}
if (jit->op_array) {
/* Only for function JIT */
_zend_jit_fix_merges(jit);
#if defined(IR_TARGET_AARCH64)
} else if (jit->trace) {
jit->ctx.deoptimization_exits = jit->trace->exit_count;
jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr;
#endif
} else {
#if defined(IR_TARGET_X86) || defined(IR_TARGET_X64)
jit->ctx.flags |= IR_GEN_CACHE_DEMOTE;
#endif
}
entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL);
if (entry) {
if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
if (str) {
ir_disasm_add_symbol(ZSTR_VAL(str), (uintptr_t)entry, size);
}
ir_disasm(str ? ZSTR_VAL(str) : "unknown",
entry, size,
(JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0,
&jit->ctx, stderr);
}
#endif
#ifndef _WIN32
if (str) {
if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) {
uintptr_t sp_offset = 0;
// ir_mem_unprotect(entry, size);
if (!(jit->ctx.flags & IR_FUNCTION)
&& zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64)
sp_offset = zend_jit_hybrid_vm_sp_adj;
#else
sp_offset = sizeof(void*);
#endif
} else {
sp_offset = sizeof(void*);
}
ir_gdb_register(ZSTR_VAL(str), entry, size, sp_offset, 0);
// ir_mem_protect(entry, size);
}
if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) {
ir_perf_map_register(ZSTR_VAL(str), entry, size);
if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) {
ir_perf_jitdump_register(ZSTR_VAL(str), entry, size);
}
}
}
#endif
}
if (jit->op_array) {
/* Only for function JIT */
const zend_op_array *op_array = jit->op_array;
zend_op *opline = (zend_op*)op_array->opcodes;
if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
while (opline->opcode == ZEND_RECV) {
opline++;
}
}
opline->handler = entry;
if (jit->ctx.entries_count) {
/* For all entries */
int i = jit->ctx.entries_count;
do {
ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]];
op_array->opcodes[insn->op2].handler = (char*)entry + insn->op3;
} while (i != 0);
}
} else {
/* Only for tracing JIT */
zend_jit_trace_info *t = jit->trace;
zend_jit_trace_stack *stack;
uint32_t i;
if (t) {
for (i = 0; i < t->stack_map_size; i++) {
stack = t->stack_map + i;
if (stack->flags & ZREG_SPILL_SLOT) {
stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP;
stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref);
}
}
}
zend_jit_trace_add_code(entry, size);
}
}
if (str) {
zend_string_release(str);
}
return entry;
}
static const void *zend_jit_trace_allocate_exit_group(uint32_t n)
{
const void *entry;
size_t size;
ir_code_buffer code_buffer;
code_buffer.start = dasm_buf;
code_buffer.end = dasm_end;
code_buffer.pos = *dasm_ptr;
entry = ir_emit_exitgroup(n, ZEND_JIT_EXIT_POINTS_PER_GROUP, zend_jit_stub_handlers[jit_stub_trace_exit],
&code_buffer, &size);
*dasm_ptr = code_buffer.pos;
if (entry) {
#ifdef HAVE_CAPSTONE
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
uint32_t i;
char name[32];
for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
snprintf(name, sizeof(name), "jit$$trace_exit_%d", n + i);
ir_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING);
}
}
#endif
}
return entry;
}
static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr));
return 1;
}
static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr)
{
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
return 1;
}
static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline)
{
uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr));
zend_jit_set_last_valid_opline(jit, opline);
return 1;
}
static bool zend_jit_guard_reference(zend_jit_ctx *jit,
const zend_op *opline,
zend_jit_addr *var_addr_ptr,
zend_jit_addr *ref_addr_ptr,
bool add_ref_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
const void *exit_addr = NULL;
ir_ref ref;
if (add_ref_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
ref = jit_Z_TYPE(jit, var_addr);
ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
}
ref = jit_Z_PTR(jit, var_addr);
*ref_addr_ptr = ZEND_ADDR_REF_ZVAL(ref);
ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
var_addr = ZEND_ADDR_REF_ZVAL(ref);
*var_addr_ptr = var_addr;
return 1;
}
static bool zend_jit_fetch_reference(zend_jit_ctx *jit,
const zend_op *opline,
uint8_t var_type,
uint32_t *var_info_ptr,
zend_jit_addr *var_addr_ptr,
bool add_ref_guard,
bool add_type_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
const void *exit_addr = NULL;
ir_ref ref;
if (add_ref_guard || add_type_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (add_ref_guard) {
ref = jit_Z_TYPE(jit, var_addr);
ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr));
}
if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
/* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper),
jit_ZVAL_ADDR(jit, var_addr));
*var_addr_ptr = var_addr;
} else {
ref = jit_Z_PTR(jit, var_addr);
ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val));
var_addr = ZEND_ADDR_REF_ZVAL(ref);
*var_addr_ptr = var_addr;
}
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (add_type_guard
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
ref = jit_Z_TYPE(jit, var_addr);
ir_GUARD(ir_EQ(ref, ir_CONST_U8(var_type)), ir_CONST_ADDR(exit_addr));
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
} else {
var_info &= ~MAY_BE_REF;
*var_info_ptr = var_info;
}
*var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
return 1;
}
static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
int32_t exit_point;
const void *exit_addr;
ir_ref ref = IR_UNUSED;
if (add_indirect_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr);
ref = jit_Z_PTR(jit, var_addr);
} else {
/* This LOAD of INDIRECT VAR, stored by the previous FETCH_(DIM/OBJ)_W,
* is eliminated by store forwarding (S2L) */
ref = jit_Z_PTR(jit, var_addr);
}
*var_info_ptr &= ~MAY_BE_INDIRECT;
var_addr = ZEND_ADDR_REF_ZVAL(ref);
*var_addr_ptr = var_addr;
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (!(var_type & IS_TRACE_REFERENCE)
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr);
//var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
}
return 1;
}
static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
{
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
size_t offset = jit_extension->offset;
const void *handler =
(zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
ir_ref ref;
zend_jit_set_ip(jit, opline);
if (GCC_GLOBAL_REGS) {
ir_CALL(IR_VOID, ir_CONST_FUNC(handler));
} else {
ref = jit_FP(jit);
ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(handler), ref);
}
if (may_throw
&& opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF) {
zend_jit_check_exception(jit);
}
while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
trace++;
}
if (!GCC_GLOBAL_REGS
&& (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_GENERATOR_CREATE) {
ir_ref addr = jit_EG(current_execute_data);
jit_STORE_FP(jit, ir_LOAD_A(addr));
}
}
if (zend_jit_trace_may_exit(op_array, opline)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_GENERATOR_CREATE) {
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
/* this check may be handled by the following OPLINE guard or jmp [IP] */
ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)),
jit_STUB_ADDR(jit, jit_stub_trace_halt));
}
} else if (GCC_GLOBAL_REGS) {
ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt));
} else {
ir_GUARD(ir_GE(ref, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt));
}
} else if (opline->opcode == ZEND_GENERATOR_RETURN ||
opline->opcode == ZEND_YIELD ||
opline->opcode == ZEND_YIELD_FROM) {
ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt));
ir_BEGIN(IR_UNUSED); /* unreachable block */
}
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) {
const zend_op *next_opline = trace->opline;
const zend_op *exit_opline = NULL;
uint32_t exit_point;
const void *exit_addr;
uint32_t old_info = 0;
uint32_t old_res_info = 0;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
if (zend_is_smart_branch(opline)) {
bool exit_if_true = 0;
exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
} else {
switch (opline->opcode) {
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_JMP_NULL:
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
exit_opline = (trace->opline == opline + 1) ?
OP_JMP_ADDR(opline, opline->op2) :
opline + 1;
break;
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
exit_opline = (trace->opline == opline + 1) ?
ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
opline + 1;
break;
}
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
}
break;
}
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
}
exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
}
break;
}
if (!exit_addr) {
return 0;
}
ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr));
}
}
zend_jit_set_last_valid_opline(jit, trace->opline);
return 1;
}
static int zend_jit_deoptimizer_start(zend_jit_ctx *jit,
zend_string *name,
uint32_t trace_num,
uint32_t exit_num)
{
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
jit->ctx.spill_base = ZREG_FP;
jit->op_array = NULL;
jit->ssa = NULL;
jit->name = zend_string_copy(name);
jit->ctx.flags |= IR_SKIP_PROLOGUE;
return 1;
}
static int zend_jit_trace_start(zend_jit_ctx *jit,
const zend_op_array *op_array,
zend_ssa *ssa,
zend_string *name,
uint32_t trace_num,
zend_jit_trace_info *parent,
uint32_t exit_num)
{
zend_jit_init_ctx(jit, (zend_jit_vm_kind == ZEND_VM_KIND_CALL) ? 0 : IR_START_BR_TARGET);
jit->ctx.spill_base = ZREG_FP;
jit->op_array = NULL;
jit->current_op_array = op_array;
jit->ssa = ssa;
jit->name = zend_string_copy(name);
if (!GCC_GLOBAL_REGS) {
if (!parent) {
ir_ref ref = ir_PARAM(IR_ADDR, "execute_data", 1);
jit_STORE_FP(jit, ref);
jit->ctx.flags |= IR_FASTCALL_FUNC;
}
}
if (parent) {
jit->ctx.flags |= IR_SKIP_PROLOGUE;
}
if (parent) {
int i;
int parent_vars_count = parent->exit_info[exit_num].stack_size;
zend_jit_trace_stack *parent_stack = parent_vars_count == 0 ? NULL :
parent->stack_map +
parent->exit_info[exit_num].stack_offset;
/* prevent clobbering of registers used for deoptimization */
for (i = 0; i < parent_vars_count; i++) {
if (STACK_FLAGS(parent_stack, i) != ZREG_CONST
&& STACK_REG(parent_stack, i) != ZREG_NONE) {
int32_t reg = STACK_REG(parent_stack, i);
ir_type type;
if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) {
type = IR_ADDR;
} else if (STACK_TYPE(parent_stack, i) == IS_LONG) {
type = IR_LONG;
} else if (STACK_TYPE(parent_stack, i) == IS_DOUBLE) {
type = IR_DOUBLE;
} else {
ZEND_UNREACHABLE();
}
if (ssa && ssa->vars[i].no_val) {
/* pass */
} else {
ir_ref ref = ir_RLOAD(type, reg);
if (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) {
/* op3 is used as a flag that the value is already stored in memory.
* In case the IR framework decides to spill the result of IR_LOAD,
* it doesn't have to store the value once again.
*
* See: insn->op3 check in ir_emit_rload()
*/
ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i));
}
}
}
}
}
if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
ZEND_ASSERT(parent->exit_info[exit_num].poly_func_reg >= 0 && parent->exit_info[exit_num].poly_this_reg >= 0);
ir_RLOAD_A(parent->exit_info[exit_num].poly_func_reg);
ir_RLOAD_A(parent->exit_info[exit_num].poly_this_reg);
}
ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num));
return 1;
}
static int zend_jit_trace_begin_loop(zend_jit_ctx *jit)
{
return ir_LOOP_BEGIN(ir_END());
}
static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi)
{
int dst_var = phi->ssa_var;
int src_var = phi->sources[0];
ir_ref ref;
ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD));
ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL);
ref = ir_PHI_2(
(jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE,
zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED);
src_var = phi->sources[1];
ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL);
jit->ra[src_var].flags |= ZREG_FORWARD;
zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref);
}
static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr)
{
if (timeout_exit_addr) {
zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
}
ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED);
ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END());
return 1;
}
static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline)
{
if (GCC_GLOBAL_REGS) {
if (!original_handler) {
ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit)));
} else {
ir_TAILCALL(IR_VOID, zend_jit_orig_opline_handler(jit));
}
} else {
if (original_handler) {
ir_ref ref;
ir_ref addr = zend_jit_orig_opline_handler(jit);
#if defined(IR_TARGET_X86)
addr = ir_CAST_FC_FUNC(addr);
#endif
ref = ir_CALL_1(IR_I32, addr, jit_FP(jit));
if (opline &&
(opline->opcode == ZEND_RETURN
|| opline->opcode == ZEND_RETURN_BY_REF
|| opline->opcode == ZEND_GENERATOR_RETURN
|| opline->opcode == ZEND_GENERATOR_CREATE
|| opline->opcode == ZEND_YIELD
|| opline->opcode == ZEND_YIELD_FROM)) {
ir_RETURN(ref);
return 1;
}
}
ir_RETURN(ir_CONST_I32(2)); // ZEND_VM_LEAVE
}
return 1;
}
static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
{
return ir_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
}
static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeout_exit_addr)
{
const void *link_addr;
/* Skip prologue. */
ZEND_ASSERT(zend_jit_trace_prologue_size != (size_t)-1);
link_addr = (const void*)((const char*)t->code_start + zend_jit_trace_prologue_size);
if (timeout_exit_addr) {
zend_jit_check_timeout(jit, NULL, timeout_exit_addr);
}
ir_IJMP(ir_CONST_ADDR(link_addr));
return 1;
}
static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_SEND_VAR:
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
return (opline->op2_type != IS_CONST) && (opline->opcode != ZEND_SEND_VAL_EX || opline->op2.num <= MAX_ARG_FLAG_NUM);
case ZEND_QM_ASSIGN:
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
return 1;
case ZEND_RETURN:
return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
case ZEND_ASSIGN:
return (opline->op1_type == IS_CV);
case ZEND_ASSIGN_OP:
if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) {
return 0;
}
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return zend_jit_supported_binary_op(opline->extended_value, op1_info, op2_info);
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
return 0;
}
if (trace && trace->op1_type != IS_UNKNOWN) {
op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
if (trace && trace->op2_type != IS_UNKNOWN) {
op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
return !(op1_info & MAY_BE_UNDEF)
&& !(op2_info & MAY_BE_UNDEF)
&& (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))
&& (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE));
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace && trace->op1_type != IS_UNKNOWN) {
op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
if (trace && trace->op2_type != IS_UNKNOWN) {
op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED));
}
return (op1_info & MAY_BE_LONG)
&& (op2_info & MAY_BE_LONG);
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
op1_info = OP1_INFO();
return opline->op1_type == IS_CV
&& (op1_info & MAY_BE_LONG)
&& !(op1_info & MAY_BE_REF);
case ZEND_STRLEN:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
case ZEND_COUNT:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
case ZEND_JMPZ:
case ZEND_JMPNZ:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (!ssa->cfg.map) {
return 0;
}
if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
return 0;
}
}
ZEND_FALLTHROUGH;
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
return 1;
case ZEND_FETCH_CONSTANT:
return 1;
case ZEND_ISSET_ISEMPTY_DIM_OBJ:
if ((opline->extended_value & ZEND_ISEMPTY)) {
return 0;
}
ZEND_FALLTHROUGH;
case ZEND_FETCH_DIM_R:
case ZEND_FETCH_DIM_IS:
case ZEND_FETCH_LIST_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace
&& trace->op1_type != IS_UNKNOWN
&& (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
}
return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
case ZEND_ASSIGN_DIM_OP:
if (opline->result_type != IS_UNUSED) {
return 0;
}
if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
return 0;
}
ZEND_FALLTHROUGH;
case ZEND_ASSIGN_DIM:
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_DIM_RW:
case ZEND_FETCH_LIST_W:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace) {
if (opline->op1_type == IS_CV) {
if ((opline->opcode == ZEND_ASSIGN_DIM
|| opline->opcode == ZEND_ASSIGN_DIM_OP)
&& (opline+1)->op1_type == IS_CV
&& (opline+1)->op1.var == opline->op1.var) {
/* skip $a[x] = $a; */
return 0;
}
} else if (opline->op1_type == IS_VAR) {
if (trace->op1_type == IS_UNKNOWN
|| !(trace->op1_type & IS_TRACE_INDIRECT)
|| opline->result_type != IS_UNUSED) {
return 0;
}
}
if (trace->op1_type != IS_UNKNOWN
&& (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
}
} else {
if (opline->op1_type != IS_CV) {
return 0;
}
}
return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING));
case ZEND_ASSIGN_OBJ_OP:
if (opline->result_type != IS_UNUSED) {
return 0;
}
if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) {
return 0;
}
ZEND_FALLTHROUGH;
case ZEND_FETCH_OBJ_R:
case ZEND_ASSIGN_OBJ:
if (opline->op2_type != IS_CONST
|| Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING
|| Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') {
return 0;
}
op1_info = OP1_INFO();
return opline->op1_type == IS_UNUSED || (op1_info & MAY_BE_OBJECT);
}
return 0;
}
static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
{
if (ssa->vars[var].no_val) {
/* we don't need the value */
return 0;
}
if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
/* Disable global register allocation,
* register allocation for SSA variables connected through Phi functions
*/
if (ssa->vars[var].definition_phi) {
return 0;
}
if (ssa->vars[var].phi_use_chain) {
zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
do {
if (!ssa->vars[phi->ssa_var].no_val) {
return 0;
}
phi = zend_ssa_next_use_phi(ssa, var, phi);
} while (phi);
}
}
if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
/* bad type */
return 0;
}
return 1;
}
static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
{
if (!zend_jit_var_supports_reg(ssa, var)) {
return 0;
}
if (ssa->vars[var].definition >= 0) {
uint32_t def = ssa->vars[var].definition;
if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
return 0;
}
}
if (ssa->vars[var].use_chain >= 0) {
int use = ssa->vars[var].use_chain;
do {
if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
return 0;
}
use = zend_ssa_next_use(ssa->ops, var, use);
} while (use >= 0);
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
int def_block, use_block, b, use, j;
zend_basic_block *bb;
zend_ssa_phi *p;
bool ret = 1;
zend_worklist worklist;
ALLOCA_FLAG(use_heap)
/* Check if live range is split by ENTRY block */
if (ssa->vars[var].definition >= 0) {
def_block =ssa->cfg.map[ssa->vars[var].definition];
} else {
ZEND_ASSERT(ssa->vars[var].definition_phi);
def_block = ssa->vars[var].definition_phi->block;
}
ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap);
if (ssa->vars[var].use_chain >= 0) {
use = ssa->vars[var].use_chain;
do {
use_block = ssa->cfg.map[use];
if (use_block != def_block) {
zend_worklist_push(&worklist, use_block);
}
use = zend_ssa_next_use(ssa->ops, var, use);
} while (use >= 0);
}
p = ssa->vars[var].phi_use_chain;
while (p) {
use_block = p->block;
if (use_block != def_block) {
bb = &ssa->cfg.blocks[use_block];
for (j = 0; j < bb->predecessors_count; j++) {
if (p->sources[j] == var) {
use_block = ssa->cfg.predecessors[bb->predecessor_offset + j];
if (use_block != def_block) {
zend_worklist_push(&worklist, use_block);
}
}
}
}
p = zend_ssa_next_use_phi(ssa, var, p);
}
while (zend_worklist_len(&worklist) != 0) {
b = zend_worklist_pop(&worklist);
bb = &ssa->cfg.blocks[b];
if (bb->flags & (ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) {
ret = 0;
break;
}
for (j = 0; j < bb->predecessors_count; j++) {
b = ssa->cfg.predecessors[bb->predecessor_offset + j];
if (b != def_block) {
zend_worklist_push(&worklist, b);
}
}
}
ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap);
return ret;
}
return 1;
}
static ir_ref jit_frameless_observer(zend_jit_ctx *jit, const zend_op *opline) {
// JIT: zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(fbc))
ir_ref observer_handler;
zend_function *fbc = ZEND_FLF_FUNC(opline);
// Not need for runtime cache or generator checks here, we just need if_unobserved
ir_ref if_unobserved = jit_observer_fcall_is_unobserved_start(jit, fbc, &observer_handler, IR_UNUSED, IR_UNUSED).if_unobserved;
// Call zend_frameless_observed_call for the main logic.
ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)zend_frameless_observed_call), jit_FP(jit));
ir_ref skip = ir_END();
ir_IF_TRUE(if_unobserved);
return skip;
}
static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline)
{
jit_SET_EX_OPLINE(jit, opline);
void *function = ZEND_FLF_HANDLER(opline);
zend_jit_addr res_addr = RES_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
zend_jit_check_exception(jit);
}
static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info)
{
jit_SET_EX_OPLINE(jit, opline);
/* Avoid dropping RC check in case op escapes. */
if (op1_info & MAY_BE_RC1) {
op1_info |= MAY_BE_RCN;
}
void *function = ZEND_FLF_HANDLER(opline);
zend_jit_addr res_addr = RES_ADDR();
zend_jit_addr op1_addr = OP1_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref);
}
if (op1_info & MAY_BE_REF) {
op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
}
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_2(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
zend_jit_check_exception(jit);
}
static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info)
{
jit_SET_EX_OPLINE(jit, opline);
/* Avoid dropping RC check in case op escapes. */
if (op1_info & MAY_BE_RC1) {
op1_info |= MAY_BE_RCN;
}
if (op2_info & MAY_BE_RC1) {
op2_info |= MAY_BE_RCN;
}
void *function = ZEND_FLF_HANDLER(opline);
zend_jit_addr res_addr = RES_ADDR();
zend_jit_addr op1_addr = OP1_ADDR();
zend_jit_addr op2_addr = OP2_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref);
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
op2_info &= ~MAY_BE_UNDEF;
op2_info |= MAY_BE_NULL;
op2_addr = ZEND_ADDR_REF_ZVAL(op2_ref);
}
if (op1_info & MAY_BE_REF) {
op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
}
if (op2_info & MAY_BE_REF) {
op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
}
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_3(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
/* Set OP1 to UNDEF in case FREE_OP2() throws. */
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) != 0 && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0) {
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
zend_jit_check_exception(jit);
}
static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, uint32_t op1_data_info)
{
jit_SET_EX_OPLINE(jit, opline);
/* Avoid dropping RC check in case op escapes. */
if (op1_info & MAY_BE_RC1) {
op1_info |= MAY_BE_RCN;
}
if (op2_info & MAY_BE_RC1) {
op2_info |= MAY_BE_RCN;
}
if (op1_data_info & MAY_BE_RC1) {
op1_data_info |= MAY_BE_RCN;
}
void *function = ZEND_FLF_HANDLER(opline);
uint8_t op_data_type = (opline + 1)->op1_type;
zend_jit_addr res_addr = RES_ADDR();
zend_jit_addr op1_addr = OP1_ADDR();
zend_jit_addr op2_addr = OP2_ADDR();
zend_jit_addr op3_addr = OP1_DATA_ADDR();
ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr);
ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr);
ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr);
ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr);
jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL);
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1);
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref);
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1);
op2_info &= ~MAY_BE_UNDEF;
op2_info |= MAY_BE_NULL;
op2_addr = ZEND_ADDR_REF_ZVAL(op2_ref);
}
if ((opline+1)->op1_type == IS_CV && (op1_data_info & MAY_BE_UNDEF)) {
op3_ref = zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1);
op1_data_info &= ~MAY_BE_UNDEF;
op1_data_info |= MAY_BE_NULL;
op3_addr = ZEND_ADDR_REF_ZVAL(op3_ref);
}
if (op1_info & MAY_BE_REF) {
op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref);
}
if (op2_info & MAY_BE_REF) {
op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref);
}
if (op1_data_info & MAY_BE_REF) {
op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref);
}
ir_ref skip_observer = IR_UNUSED;
if (ZEND_OBSERVER_ENABLED) {
skip_observer = jit_frameless_observer(jit, opline);
}
ir_CALL_4(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref, op3_ref);
if (skip_observer != IR_UNUSED) {
ir_MERGE_WITH(skip_observer);
}
jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL);
/* Set OP1 to UNDEF in case FREE_OP2() throws. */
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
|| (op_data_type & (IS_VAR|IS_TMP_VAR)))) {
jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF);
}
jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL);
/* If OP1 is a TMP|VAR, we don't need to set OP2 to UNDEF on free because
* zend_fetch_debug_backtrace aborts when it encounters the first UNDEF TMP|VAR. */
if (!(opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0
&& (op_data_type & (IS_VAR|IS_TMP_VAR)) != 0) {
jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF);
}
jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL);
zend_jit_check_exception(jit);
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* indent-tabs-mode: t
* End:
*/
Reference in New Issue View Git Blame Copy Permalink