php/archived-php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2026-03-24 00:02:20 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
be8589651da52ce132e4e9c4a9c2ebcc23824c94
archived-php-src/ext/opcache/jit/zend_jit_arm64.dasc
Dmitry Stogov be8589651d Merge branch 'PHP-8.2' into PHP-8.3 
* PHP-8.2:
  Fix GH-15652: Segmentation fault in the Zend engine when JIT enabled (#15717)
2024-09-02 21:55:40 +03:00

15608 lines
472 KiB
Plaintext
Raw Blame History

/*
* +----------------------------------------------------------------------+
* | Zend JIT |
* +----------------------------------------------------------------------+
* | Copyright (c) The PHP Group |
* +----------------------------------------------------------------------+
* | This source file is subject to version 3.01 of the PHP license, |
* | that is bundled with this package in the file LICENSE, and is |
* | available through the world-wide-web at the following url: |
* | https://www.php.net/license/3_01.txt |
* | If you did not receive a copy of the PHP license and are unable to |
* | obtain it through the world-wide-web, please send a note to |
* | license@php.net so we can mail you a copy immediately. |
* +----------------------------------------------------------------------+
* | Authors: Dmitry Stogov <dmitry@php.net> |
* | Xinchen Hui <laruence@php.net> |
* | Hao Sun <hao.sun@arm.com> |
* +----------------------------------------------------------------------+
*/
|.arch arm64
|.define FP, x27
|.define IP, x28
|.define IPl, w28
|.define RX, x28 // the same as VM IP reused as a general purpose reg
|.define LR, x30
|.define CARG1, x0
|.define CARG2, x1
|.define CARG3, x2
|.define CARG4, x3
|.define CARG5, x4
|.define CARG6, x5
|.define CARG1w, w0
|.define CARG2w, w1
|.define CARG3w, w2
|.define CARG4w, w3
|.define CARG5w, w4
|.define CARG6w, w5
|.define RETVALx, x0
|.define RETVALw, w0
|.define FCARG1x, x0
|.define FCARG1w, w0
|.define FCARG2x, x1
|.define FCARG2w, w1
|.define SPAD, 0x20 // padding for CPU stack alignment
|.define NR_SPAD, 0x30 // padding for CPU stack alignment
|.define T3, [sp, #0x28] // Used to store old value of IP (CALL VM only)
|.define T2, [sp, #0x20] // Used to store old value of FP (CALL VM only)
|.define T1, [sp, #0x10]
// We use REG0/1/2 and FPR0/1 to replace r0/1/2 and xmm0/1 in the x86 implementation.
// Scratch registers
|.define REG0, x8
|.define REG0w, w8
|.define REG1, x9
|.define REG1w, w9
|.define REG2, x10
|.define REG2w, w10
|.define FPR0, d0
|.define FPR1, d1
|.define ZREG_REG0, ZREG_X8
|.define ZREG_REG1, ZREG_X9
|.define ZREG_REG2, ZREG_X10
|.define ZREG_FPR0, ZREG_V0
|.define ZREG_FPR1, ZREG_V1
// Temporaries, not preserved across calls
|.define TMP1, x15
|.define TMP1w, w15
|.define TMP2, x16
|.define TMP2w, w16
|.define TMP3, x17 // TODO: remember about hard-coded: mrs TMP3, tpidr_el0
|.define TMP3w, w17
|.define FPTMP, d16
|.define ZREG_TMP1, ZREG_X15
|.define ZREG_TMP2, ZREG_X16
|.define ZREG_TMP3, ZREG_X17
|.define ZREG_FPTMP, ZREG_V16
|.define HYBRID_SPAD, 32 // padding for stack alignment
#define TMP_ZVAL_OFFSET 16
#define DASM_ALIGNMENT 16
const char* zend_reg_name[] = {
"x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
"x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
"x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
"x24", "x25", "x26", "x27", "x28", "x29", "x30", "sp",
"d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
"d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15",
"d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23",
"d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"
};
#define ZREG_FCARG1 ZREG_X0
#define ZREG_FCARG2 ZREG_X1
|.type EX, zend_execute_data, FP
|.type OP, zend_op
|.type ZVAL, zval
|.actionlist dasm_actions
|.globals zend_lb
|.section code, cold_code, jmp_table
static void* dasm_labels[zend_lb_MAX];
#if ZTS
static size_t tsrm_ls_cache_tcb_offset = 0;
static size_t tsrm_tls_index = 0;
static size_t tsrm_tls_offset = 0;
# ifdef __APPLE__
struct TLVDescriptor {
void* (*thunk)(struct TLVDescriptor*);
uint64_t key;
uint64_t offset;
};
typedef struct TLVDescriptor TLVDescriptor;
# elif defined(__FreeBSD__)
/* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */
typedef struct TLSDescriptor {
void* thunk;
int index;
size_t offset;
} TLSDescriptor;
# endif
#endif
#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1)))
/* Encoding of immediate. */
#define MAX_IMM12 0xfff // maximum value for imm12
#define MAX_IMM16 0xffff // maximum value for imm16
#define MOVZ_IMM MAX_IMM16 // movz insn
#define LDR_STR_PIMM64 (MAX_IMM12*8) // ldr/str insn for 64-bit register: pimm is imm12 * 8
#define LDR_STR_PIMM32 (MAX_IMM12*4) // ldr/str insn for 32-bit register: pimm is imm12 * 4
#define LDRB_STRB_PIMM MAX_IMM12 // ldrb/strb insn
#define B_IMM (1<<27) // signed imm26 * 4
#define ADR_IMM (1<<20) // signed imm21
#define ADRP_IMM (1LL<<32) // signed imm21 * 4096
static bool arm64_may_use_b(const void *addr)
{
if (addr >= dasm_buf && addr < dasm_end) {
return (((char*)dasm_end - (char*)dasm_buf) < B_IMM);
} else if (addr >= dasm_end) {
return (((char*)addr - (char*)dasm_buf) < B_IMM);
} else if (addr < dasm_buf) {
return (((char*)dasm_end - (char*)addr) < B_IMM);
}
return 0;
}
static bool arm64_may_use_adr(const void *addr)
{
if (addr >= dasm_buf && addr < dasm_end) {
return (((char*)dasm_end - (char*)dasm_buf) < ADR_IMM);
} else if (addr >= dasm_end) {
return (((char*)addr - (char*)dasm_buf) < ADR_IMM);
} else if (addr < dasm_buf) {
return (((char*)dasm_end - (char*)addr) < ADR_IMM);
}
return 0;
}
static bool arm64_may_use_adrp(const void *addr)
{
if (addr >= dasm_buf && addr < dasm_end) {
return (((char*)dasm_end - (char*)dasm_buf) < ADRP_IMM);
} else if (addr >= dasm_end) {
return (((char*)addr - (char*)dasm_buf) < ADRP_IMM);
} else if (addr < dasm_buf) {
return (((char*)dasm_end - (char*)addr) < ADRP_IMM);
}
return 0;
}
/* Determine whether "val" falls into two allowed ranges:
* Range 1: [0, 0xfff]
* Range 2: LSL #12 to Range 1
* Used to guard the immediate encoding for add/adds/sub/subs/cmp/cmn instructions. */
static bool arm64_may_encode_imm12(const int64_t val)
{
return (val >= 0 && (val <= MAX_IMM12 || !(val & 0xffffffffff000fff)));
}
/* Determine whether an immediate value can be encoded as the immediate operand of logical instructions. */
static bool logical_immediate_p(uint64_t value, uint32_t reg_size)
{
/* fast path: power of two */
if (value > 0 && !(value & (value - 1))) {
return true;
}
if (reg_size == 32) {
if (dasm_imm13((uint32_t)value, (uint32_t)value) != -1) {
return true;
}
} else if (reg_size == 64) {
if (dasm_imm13((uint32_t)value, (uint32_t)(value >> 32)) != -1) {
return true;
}
} else {
ZEND_UNREACHABLE();
}
return false;
}
/* Not Implemented Yet */
|.macro NIY
|| //ZEND_ASSERT(0);
| brk #0
|.endmacro
|.macro NIY_STUB
|| //ZEND_ASSERT(0);
| brk #0
|.endmacro
|.macro ADD_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| add sp, sp, # HYBRID_SPAD
||#endif
|.endmacro
|.macro SUB_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| sub sp, sp, # HYBRID_SPAD
||#endif
|.endmacro
/* Move address into register. TODO: Support 52-bit address */
|.macro LOAD_ADDR, reg, addr
| // 48-bit virtual address
|| if (((uintptr_t)(addr)) == 0) {
| mov reg, xzr
|| } else if (((uintptr_t)(addr)) <= MOVZ_IMM) {
| movz reg, #((uint64_t)(addr))
|| } else if (arm64_may_use_adr((void*)(addr))) {
| adr reg, &addr
|| } else if (arm64_may_use_adrp((void*)(addr))) {
| adrp reg, &(((uintptr_t)(addr)))
|| if (((uintptr_t)(addr)) & 0xfff) {
| add reg, reg, #(((uintptr_t)(addr)) & 0xfff)
|| }
|| } else if ((uintptr_t)(addr) & 0xffff) {
| movz reg, #((uintptr_t)(addr) & 0xffff)
|| if (((uintptr_t)(addr) >> 16) & 0xffff) {
| movk reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16
|| }
|| if (((uintptr_t)(addr) >> 32) & 0xffff) {
| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32
|| }
|| } else if (((uintptr_t)(addr) >> 16) & 0xffff) {
| movz reg, #(((uintptr_t)(addr) >> 16) & 0xffff), lsl #16
|| if (((uintptr_t)(addr) >> 32) & 0xffff) {
| movk reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32
|| }
|| } else {
| movz reg, #(((uintptr_t)(addr) >> 32) & 0xffff), lsl #32
|| }
|.endmacro
/* Move 32-bit immediate value into register. */
|.macro LOAD_32BIT_VAL, reg, val
|| if (((uint32_t)(val)) <= MOVZ_IMM) {
| movz reg, #((uint32_t)(val))
|| } else if (((uint32_t)(val) & 0xffff)) {
| movz reg, #((uint32_t)(val) & 0xffff)
|| if ((((uint32_t)(val) >> 16) & 0xffff)) {
| movk reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16
|| }
|| } else {
| movz reg, #(((uint32_t)(val) >> 16) & 0xffff), lsl #16
|| }
|.endmacro
/* Move 64-bit immediate value into register. */
|.macro LOAD_64BIT_VAL, reg, val
|| if (((uint64_t)(val)) == 0) {
| mov reg, xzr
|| } else if (((uint64_t)(val)) <= MOVZ_IMM) {
| movz reg, #((uint64_t)(val))
|| } else if (~((uint64_t)(val)) <= MOVZ_IMM) {
| movn reg, #(~((uint64_t)(val)))
|| } else if ((uint64_t)(val) & 0xffff) {
| movz reg, #((uint64_t)(val) & 0xffff)
|| if (((uint64_t)(val) >> 16) & 0xffff) {
| movk reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16
|| }
|| if (((uint64_t)(val) >> 32) & 0xffff) {
| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32
|| }
|| if ((((uint64_t)(val) >> 48) & 0xffff)) {
| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
|| }
|| } else if (((uint64_t)(val) >> 16) & 0xffff) {
| movz reg, #(((uint64_t)(val) >> 16) & 0xffff), lsl #16
|| if (((uint64_t)(val) >> 32) & 0xffff) {
| movk reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32
|| }
|| if ((((uint64_t)(val) >> 48) & 0xffff)) {
| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
|| }
|| } else if (((uint64_t)(val) >> 32) & 0xffff) {
| movz reg, #(((uint64_t)(val) >> 32) & 0xffff), lsl #32
|| if ((((uint64_t)(val) >> 48) & 0xffff)) {
| movk reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
|| }
|| } else {
| movz reg, #(((uint64_t)(val) >> 48) & 0xffff), lsl #48
|| }
|.endmacro
/* Extract the low 8 bits from 'src_reg' into 'dst_reg'.
* Note: 0xff can be encoded as imm for 'and' instruction. */
|.macro GET_LOW_8BITS, dst_reg, src_reg
| and dst_reg, src_reg, #0xff
|.endmacro
/* Bitwise operation with immediate. 'bw_ins' can be and/orr/eor/ands.
* 32-bit and 64-bit registers are distinguished. */
|.macro BW_OP_32_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg
|| if (val == 0) {
| bw_ins dst_reg, src_reg1, wzr
|| } else if (logical_immediate_p((uint32_t)val, 32)) {
| bw_ins dst_reg, src_reg1, #val
|| } else {
| LOAD_32BIT_VAL tmp_reg, val
| bw_ins dst_reg, src_reg1, tmp_reg
|| }
|.endmacro
|.macro BW_OP_64_WITH_CONST, bw_ins, dst_reg, src_reg1, val, tmp_reg
|| if (val == 0) {
| bw_ins dst_reg, src_reg1, xzr
|| } else if (logical_immediate_p(val, 64)) {
| bw_ins dst_reg, src_reg1, #val
|| } else {
| LOAD_64BIT_VAL tmp_reg, val
| bw_ins dst_reg, src_reg1, tmp_reg
|| }
|.endmacro
/* Test bits 'tst' with immediate. 32-bit and 64-bit registers are distinguished. */
|.macro TST_32_WITH_CONST, reg, val, tmp_reg
|| if (val == 0) {
| tst reg, wzr
|| } else if (logical_immediate_p((uint32_t)val, 32)) {
| tst reg, #val
|| } else {
| LOAD_32BIT_VAL tmp_reg, val
| tst reg, tmp_reg
|| }
|.endmacro
|.macro TST_64_WITH_CONST, reg, val, tmp_reg
|| if (val == 0) {
| tst reg, xzr
|| } else if (logical_immediate_p(val, 64)) {
| tst reg, #val
|| } else {
| LOAD_64BIT_VAL tmp_reg, val
| tst reg, tmp_reg
|| }
|.endmacro
/* Test bits between 64-bit register with constant 1. */
|.macro TST_64_WITH_ONE, reg
| tst reg, #1
|.endmacro
/* Compare a register value with immediate. 32-bit and 64-bit registers are distinguished.
* Note: Comparing 64-bit register with 32-bit immediate is handled in CMP_64_WITH_CONST_32. */
|.macro CMP_32_WITH_CONST, reg, val, tmp_reg
|| if (val == 0) {
| cmp reg, wzr
|| } else if (arm64_may_encode_imm12((int64_t)(val))) {
| cmp reg, #val
|| } else if (arm64_may_encode_imm12((int64_t)(-val))) {
| cmn reg, #-val
|| } else {
| LOAD_32BIT_VAL tmp_reg, val
| cmp reg, tmp_reg
|| }
|.endmacro
|.macro CMP_64_WITH_CONST_32, reg, val, tmp_reg
|| if (val == 0) {
| cmp reg, xzr
|| } else if (arm64_may_encode_imm12((int64_t)(val))) {
| cmp reg, #val
|| } else if (arm64_may_encode_imm12((int64_t)(-val))) {
| cmn reg, #-val
|| } else {
| LOAD_32BIT_VAL tmp_reg, val
| cmp reg, tmp_reg
|| }
|.endmacro
|.macro CMP_64_WITH_CONST, reg, val, tmp_reg
|| if (val == 0) {
| cmp reg, xzr
|| } else if (arm64_may_encode_imm12((int64_t)(val))) {
| cmp reg, #val
|| } else if (arm64_may_encode_imm12((int64_t)(-val))) {
| cmn reg, #-val
|| } else {
| LOAD_64BIT_VAL tmp_reg, val
| cmp reg, tmp_reg
|| }
|.endmacro
/* Add/sub a register value with immediate. 'add_sub_ins' can be add/sub/adds/subs.
* 32-bit and 64-bit registers are distinguished.
* Note: Case of 64-bit register with 32-bit immediate is handled in ADD_SUB_64_WITH_CONST_32. */
|.macro ADD_SUB_32_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg
|| if (val == 0) {
| add_sub_ins dst_reg, src_reg1, wzr
|| } else if (arm64_may_encode_imm12((int64_t)(val))) {
| add_sub_ins dst_reg, src_reg1, #val
|| } else {
| LOAD_32BIT_VAL tmp_reg, val
| add_sub_ins dst_reg, src_reg1, tmp_reg
|| }
|.endmacro
|.macro ADD_SUB_64_WITH_CONST_32, add_sub_ins, dst_reg, src_reg1, val, tmp_reg
|| if (val == 0) {
| add_sub_ins dst_reg, src_reg1, xzr
|| } else if (arm64_may_encode_imm12((int64_t)(val))) {
| add_sub_ins dst_reg, src_reg1, #val
|| } else {
| LOAD_32BIT_VAL tmp_reg, val
| add_sub_ins dst_reg, src_reg1, tmp_reg
|| }
|.endmacro
|.macro ADD_SUB_64_WITH_CONST, add_sub_ins, dst_reg, src_reg1, val, tmp_reg
|| if (val == 0) {
| add_sub_ins dst_reg, src_reg1, xzr
|| } else if (arm64_may_encode_imm12((int64_t)(val))) {
| add_sub_ins dst_reg, src_reg1, #val
|| } else {
| LOAD_64BIT_VAL tmp_reg, val
| add_sub_ins dst_reg, src_reg1, tmp_reg
|| }
|.endmacro
/* Memory access(load/store) with 32-bit 'offset'.
* Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register.
* 8-bit, 32-bit and 64-bit registers to be transferred are distinguished.
* Note: 'reg' can be used as 'tmp_reg' if 1) 'reg' is one GPR, AND 2) 'reg' != 'base_reg', AND 3) ins is 'ldr'. */
|.macro MEM_ACCESS_64_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg
|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) {
| LOAD_32BIT_VAL tmp_reg, offset
| ldr_str_ins reg, [base_reg, tmp_reg]
|| } else {
| ldr_str_ins reg, [base_reg, #(offset)]
|| }
|.endmacro
|.macro MEM_ACCESS_32_WITH_UOFFSET, ldr_str_ins, reg, base_reg, offset, tmp_reg
|| if (((uintptr_t)(offset)) > LDR_STR_PIMM32) {
| LOAD_32BIT_VAL tmp_reg, offset
| ldr_str_ins reg, [base_reg, tmp_reg]
|| } else {
| ldr_str_ins reg, [base_reg, #(offset)]
|| }
|.endmacro
|.macro MEM_ACCESS_8_WITH_UOFFSET, ldrb_strb_ins, reg, base_reg, offset, tmp_reg
|| if (((uintptr_t)(offset)) > LDRB_STRB_PIMM) {
| LOAD_32BIT_VAL tmp_reg, offset
| ldrb_strb_ins reg, [base_reg, tmp_reg]
|| } else {
| ldrb_strb_ins reg, [base_reg, #(offset)]
|| }
|.endmacro
/* Memory access(load/store) with 64-bit 'offset'.
* Use "unsigned offset" variant if 'offset' can be encoded, otherwise move 'offset' into temp register. */
|.macro MEM_ACCESS_64_WITH_UOFFSET_64, ldr_str_ins, reg, base_reg, offset, tmp_reg
|| if (((uintptr_t)(offset)) > LDR_STR_PIMM64) {
| LOAD_64BIT_VAL tmp_reg, offset
| ldr_str_ins reg, [base_reg, tmp_reg]
|| } else {
| ldr_str_ins reg, [base_reg, #(offset)]
|| }
|.endmacro
/* ZTS: get thread local variable "_tsrm_ls_cache" */
|.macro LOAD_TSRM_CACHE, reg
||#ifdef __APPLE__
| .long 0xd53bd071 // TODO: hard-coded: mrs TMP3, tpidrro_el0
| and TMP3, TMP3, #0xfffffffffffffff8
| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->key << 3), TMP1
| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, (((TLVDescriptor*)tsrm_ls_cache_tcb_offset)->offset), TMP1
||#else
| .long 0xd53bd051 // TODO: hard-coded: mrs TMP3, tpidr_el0
|| if (tsrm_ls_cache_tcb_offset == 0) {
| ldr TMP3, [TMP3, #0]
| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, TMP3, TMP3, tsrm_tls_index, TMP1
| MEM_ACCESS_64_WITH_UOFFSET_64 ldr, reg, TMP3, tsrm_tls_offset, TMP1
|| } else {
|| ZEND_ASSERT(tsrm_ls_cache_tcb_offset <= LDR_STR_PIMM64);
| ldr reg, [TMP3, #tsrm_ls_cache_tcb_offset]
|| }
||#endif
|.endmacro
|.macro LOAD_ADDR_ZTS, reg, struct, field
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| ADD_SUB_64_WITH_CONST_32 add, reg, TMP3, (struct.._offset + offsetof(zend_..struct, field)), reg
| .else
| LOAD_ADDR reg, &struct.field
| .endif
|.endmacro
/* Store address 'addr' into memory 'mem'. */
|.macro ADDR_STORE, mem, addr, tmp_reg
| LOAD_ADDR tmp_reg, addr
| str tmp_reg, mem
|.endmacro
/* Store a register value 'reg' into memory 'addr'.
* For ZTS mode, base register with unsigned offset variant is used,
* and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */
|.macro MEM_STORE, str_ins, reg, addr, tmp_reg
|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) {
| adr tmp_reg, &addr
| str_ins reg, [tmp_reg]
|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) {
| adrp tmp_reg, &(((uintptr_t)(addr)))
| str_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)]
|| } else {
| LOAD_ADDR tmp_reg, addr
| str_ins reg, [tmp_reg]
|| }
|.endmacro
|.macro MEM_STORE_64_ZTS, str_ins, reg, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| MEM_ACCESS_64_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
| .else
| MEM_STORE str_ins, reg, &struct.field, tmp_reg
| .endif
|.endmacro
|.macro MEM_STORE_32_ZTS, str_ins, reg, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| MEM_ACCESS_32_WITH_UOFFSET str_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
| .else
| MEM_STORE str_ins, reg, &struct.field, tmp_reg
| .endif
|.endmacro
|.macro MEM_STORE_8_ZTS, strb_ins, reg, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| MEM_ACCESS_8_WITH_UOFFSET strb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
| .else
| MEM_STORE strb_ins, reg, &struct.field, tmp_reg
| .endif
|.endmacro
/* Load value from memory 'addr' and write it into register 'reg'.
* For ZTS mode, base register with unsigned offset variant is used,
* and 8-bit/32-bit/64-bit registers to be transferred are distinguished. */
|.macro MEM_LOAD, ldr_ins, reg, addr, tmp_reg
|| if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adr((void*)(addr))) {
| adr tmp_reg, &addr
| ldr_ins reg, [tmp_reg]
|| } else if (((uintptr_t)(addr)) > MOVZ_IMM && arm64_may_use_adrp((void*)(addr))) {
| adrp tmp_reg, &(((uintptr_t)(addr)))
| ldr_ins reg, [tmp_reg, #(((uintptr_t)(addr)) & 0xfff)]
|| } else {
| LOAD_ADDR tmp_reg, addr
| ldr_ins reg, [tmp_reg]
|| }
|.endmacro
|.macro MEM_LOAD_64_ZTS, ldr_ins, reg, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
| .else
| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg
| .endif
|.endmacro
|.macro MEM_LOAD_32_ZTS, ldr_ins, reg, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| MEM_ACCESS_32_WITH_UOFFSET ldr_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
| .else
| MEM_LOAD ldr_ins, reg, &struct.field, tmp_reg
| .endif
|.endmacro
|.macro MEM_LOAD_8_ZTS, ldrb_ins, reg, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| MEM_ACCESS_8_WITH_UOFFSET ldrb_ins, reg, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
| .else
| MEM_LOAD ldrb_ins, reg, &struct.field, tmp_reg
| .endif
|.endmacro
/* Conduct arithmetic operation between the value in memory 'addr' and register value in 'reg',
* and the computation result is stored back in 'reg'. 'op_ins' can be add/sub. */
|.macro MEM_LOAD_OP, op_ins, ldr_ins, reg, addr, tmp_reg1, tmp_reg2
| MEM_LOAD ldr_ins, tmp_reg1, addr, tmp_reg2
| op_ins reg, reg, tmp_reg1
|.endmacro
|.macro MEM_LOAD_OP_ZTS, op_ins, ldr_ins, reg, struct, field, tmp_reg1, tmp_reg2
| .if ZTS
| LOAD_TSRM_CACHE TMP3
| MEM_ACCESS_64_WITH_UOFFSET ldr_ins, tmp_reg2, TMP3, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg1
| op_ins reg, reg, tmp_reg2
| .else
| MEM_LOAD_OP op_ins, ldr_ins, reg, &struct.field, tmp_reg1, tmp_reg2
| .endif
|.endmacro
/* Conduct arithmetic operation between the value in memory 'addr' and operand 'op', and the computation
* result is stored back to memory 'addr'. Operand 'op' can be either a register value or an immediate value.
* Currently, only add instruction is used as 'op_ins'.
* Note: It should be guaranteed that the immediate value can be encoded for 'op_ins'. */
|.macro MEM_UPDATE, op_ins, ldr_ins, str_ins, op, addr, tmp_reg1, tmp_reg2
| LOAD_ADDR tmp_reg2, addr
| ldr_ins, tmp_reg1, [tmp_reg2]
| op_ins tmp_reg1, tmp_reg1, op
| str_ins tmp_reg1, [tmp_reg2]
|.endmacro
|.macro MEM_UPDATE_ZTS, op_ins, ldr_ins, str_ins, op, struct, field, tmp_reg1, tmp_reg2
| .if ZTS
| LOAD_TSRM_CACHE TMP3
|| if (((uintptr_t)(struct.._offset+offsetof(zend_..struct, field))) > LDR_STR_PIMM64) {
| LOAD_32BIT_VAL tmp_reg1, (struct.._offset+offsetof(zend_..struct, field))
| ldr_ins tmp_reg2, [TMP3, tmp_reg1]
| op_ins tmp_reg2, tmp_reg2, op
| str_ins tmp_reg2, [TMP3, tmp_reg1]
|| } else {
| ldr_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))]
| op_ins tmp_reg2, tmp_reg2, op
| str_ins tmp_reg2, [TMP3, #(struct.._offset+offsetof(zend_..struct, field))]
|| }
| .else
| MEM_UPDATE op_ins, ldr_ins, str_ins, op, &struct.field, tmp_reg1, tmp_reg2
| .endif
|.endmacro
|.macro EXT_CALL, func, tmp_reg
|| if (arm64_may_use_b(func)) {
| bl &func
|| } else {
| LOAD_ADDR tmp_reg, func
| blr tmp_reg
|| }
|.endmacro
|.macro EXT_JMP, func, tmp_reg
|| if (arm64_may_use_b(func)) {
| b &func
|| } else {
| LOAD_ADDR tmp_reg, func
| br tmp_reg
|| }
|.endmacro
|.macro SAVE_IP
|| if (GCC_GLOBAL_REGS) {
| str IP, EX->opline
|| }
|.endmacro
|.macro LOAD_IP
|| if (GCC_GLOBAL_REGS) {
| ldr IP, EX->opline
|| }
|.endmacro
|.macro LOAD_IP_ADDR, addr
|| if (GCC_GLOBAL_REGS) {
| LOAD_ADDR IP, addr
|| } else {
| ADDR_STORE EX->opline, addr, RX
|| }
|.endmacro
|.macro LOAD_IP_ADDR_ZTS, struct, field, tmp_reg
| .if ZTS
|| if (GCC_GLOBAL_REGS) {
| LOAD_TSRM_CACHE IP
| ADD_SUB_64_WITH_CONST_32 add, IP, IP, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
|| } else {
| LOAD_TSRM_CACHE RX
| ADD_SUB_64_WITH_CONST_32 add, RX, RX, (struct.._offset+offsetof(zend_..struct, field)), tmp_reg
| str RX, EX->opline
|| }
| .else
| LOAD_IP_ADDR &struct.field
| .endif
|.endmacro
|.macro GET_IP, reg
|| if (GCC_GLOBAL_REGS) {
| mov reg, IP
|| } else {
| ldr reg, EX->opline
|| }
|.endmacro
/* Update IP with register 'reg'. Note: shift variant is handled by ADD_IP_SHIFT. */
|.macro ADD_IP, reg, tmp_reg
|| if (GCC_GLOBAL_REGS) {
| add IP, IP, reg
|| } else {
| ldr tmp_reg, EX->opline
| add tmp_reg, tmp_reg, reg
| str tmp_reg, EX->opline
|| }
|.endmacro
|.macro ADD_IP_SHIFT, reg, shift, tmp_reg
|| if (GCC_GLOBAL_REGS) {
| add IP, IP, reg, shift
|| } else {
| ldr tmp_reg, EX->opline
| add tmp_reg, tmp_reg, reg, shift
| str tmp_reg, EX->opline
|| }
|.endmacro
/* Update IP with 32-bit immediate 'val'. */
|.macro ADD_IP_WITH_CONST, val, tmp_reg
|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val)));
|| if (GCC_GLOBAL_REGS) {
| add IP, IP, #val
|| } else {
| ldr tmp_reg, EX->opline
| add tmp_reg, tmp_reg, #val
| str tmp_reg, EX->opline
|| }
|.endmacro
|.macro JMP_IP, tmp_reg
|| if (GCC_GLOBAL_REGS) {
| ldr tmp_reg, [IP]
| br tmp_reg
|| } else {
| ldr tmp_reg, EX:CARG1->opline
| br tmp_reg
|| }
|.endmacro
|.macro CMP_IP, addr, tmp_reg1, tmp_reg2
| LOAD_ADDR tmp_reg1, addr
|| if (GCC_GLOBAL_REGS) {
| cmp IP, tmp_reg1
|| } else {
| ldr tmp_reg2, EX->opline
| cmp tmp_reg2, tmp_reg1
|| }
|.endmacro
|.macro LOAD_ZVAL_ADDR, reg, addr
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| LOAD_ADDR reg, Z_ZV(addr)
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|| if (Z_OFFSET(addr)) {
| ADD_SUB_64_WITH_CONST_32 add, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), reg
|| } else {
|| if (Z_REG(addr) == ZREG_RSP) {
| mov reg, sp
|| } else {
| mov reg, Rx(Z_REG(addr))
|| }
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro GET_Z_TYPE_INFO, reg, zv
| ldr reg, [zv, #offsetof(zval,u1.type_info)]
|.endmacro
|.macro SET_Z_TYPE_INFO, zv, type, tmp_reg
| LOAD_32BIT_VAL tmp_reg, type
| str tmp_reg, [zv, #offsetof(zval,u1.type_info)]
|.endmacro
|.macro GET_ZVAL_TYPE, reg, addr, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_8_WITH_UOFFSET ldrb, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.v.type), tmp_reg
|.endmacro
|.macro GET_ZVAL_TYPE_INFO, reg, addr, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_32_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg
|.endmacro
|.macro SET_ZVAL_TYPE_INFO, addr, type, tmp_reg1, tmp_reg2
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| LOAD_32BIT_VAL tmp_reg1, type
| MEM_ACCESS_32_WITH_UOFFSET str, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg2
|.endmacro
|.macro SET_ZVAL_TYPE_INFO_FROM_REG, addr, reg, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_32_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval,u1.type_info), tmp_reg
|.endmacro
|.macro GET_Z_PTR, reg, zv
| ldr reg, [zv]
|.endmacro
|.macro GET_ZVAL_PTR, reg, addr, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_64_WITH_UOFFSET ldr, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|.endmacro
|.macro SET_ZVAL_PTR, addr, reg, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|.endmacro
|.macro UNDEF_OPLINE_RESULT, tmp_reg
| ldr REG0, EX->opline
| ldr REG0w, OP:REG0->result.var
| add REG0, FP, REG0
| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg
|.endmacro
|.macro UNDEF_OPLINE_RESULT_IF_USED, tmp_reg1, tmp_reg2
| ldrb tmp_reg1, OP:RX->result_type
| TST_32_WITH_CONST tmp_reg1, (IS_TMP_VAR|IS_VAR), tmp_reg2
| beq >1
| ldr REG0w, OP:RX->result.var
| add REG0, FP, REG0
| SET_Z_TYPE_INFO REG0, IS_UNDEF, tmp_reg1
|1:
|.endmacro
/* Floating-point comparison between register 'reg' and value from memory 'addr'.
* Note: the equivalent macros in JIT/x86 are SSE_AVX_OP and SSE_OP. */
|.macro DOUBLE_CMP, reg, addr, tmp_reg, fp_tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| MEM_LOAD ldr, Rd(fp_tmp_reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg)
| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0)
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(fp_tmp_reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg)
| fcmp Rd(reg-ZREG_V0), Rd(fp_tmp_reg-ZREG_V0)
|| } else if (Z_MODE(addr) == IS_REG) {
| fcmp Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
/* Convert LONG value 'val' into DOUBLE type, and move it into FP register 'reg'.
* Note: the equivalent macro in JIT/x86 is SSE_GET_LONG. */
|.macro DOUBLE_GET_LONG, reg, val, tmp_reg
|| if (val == 0) {
| fmov Rd(reg-ZREG_V0), xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64
|| } else {
| LOAD_64BIT_VAL Rx(tmp_reg), val
| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg)
|| }
|.endmacro
/* Convert LONG value from memory 'addr' into DOUBLE type, and move it into FP register 'reg'.
* Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_LVAL. */
|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg1, tmp_reg2
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg1
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg2)
| scvtf Rd(reg-ZREG_V0), Rx(tmp_reg1)
|| } else if (Z_MODE(addr) == IS_REG) {
| scvtf Rd(reg-ZREG_V0), Rx(Z_REG(addr))
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
/* Floating-point arithmetic operation between two FP registers.
* Note: the equivalent macro in JIT/x86 is AVX_MATH_REG. */
|.macro DOUBLE_MATH_REG, opcode, dst_reg, op1_reg, op2_reg
|| switch (opcode) {
|| case ZEND_ADD:
| fadd Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
|| break;
|| case ZEND_SUB:
| fsub Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
|| break;
|| case ZEND_MUL:
| fmul Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
|| break;
|| case ZEND_DIV:
| fdiv Rd(dst_reg-ZREG_V0), Rd(op1_reg-ZREG_V0), Rd(op2_reg-ZREG_V0)
|| break;
|| }
|.endmacro
/* Conduct binary operation between register 'reg' and value from memory 'addr',
* and the computation result is stored in 'reg'.
* For LONG_ADD_SUB, 'add_sub_ins' can be adds/subs. For LONG_BW_OP, 'bw_ins' can be and/orr/eor.
* For LONG_CMP, 'cmp' instruction is used by default and only flag registers are affected.
* Note: the equivalent macro in JIT/x86 is LONG_OP. */
|.macro LONG_ADD_SUB, add_sub_ins, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| ADD_SUB_64_WITH_CONST add_sub_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
| add_sub_ins Rx(reg), Rx(reg), tmp_reg
|| } else if (Z_MODE(addr) == IS_REG) {
| add_sub_ins Rx(reg), Rx(reg), Rx(Z_REG(addr))
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_BW_OP, bw_ins, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| BW_OP_64_WITH_CONST bw_ins, Rx(reg), Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
| bw_ins Rx(reg), Rx(reg), tmp_reg
|| } else if (Z_MODE(addr) == IS_REG) {
| bw_ins Rx(reg), Rx(reg), Rx(Z_REG(addr))
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_CMP, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| CMP_64_WITH_CONST Rx(reg), Z_LVAL_P(Z_ZV(addr)), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
| cmp Rx(reg), tmp_reg
|| } else if (Z_MODE(addr) == IS_REG) {
| cmp Rx(reg), Rx(Z_REG(addr))
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
/* Conduct add/sub between value from memory 'addr' and an immediate value 'val', and
* the computation result is stored back into 'addr'.
* Note: it should be guaranteed that 'val' can be encoded into add/sub instruction. */
|.macro LONG_ADD_SUB_WITH_IMM, add_sub_ins, addr, val, tmp_reg1, tmp_reg2
|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(val)));
|| if (Z_MODE(addr) == IS_MEM_ZVAL) {
|| if (((uint32_t)(Z_OFFSET(addr))) > LDR_STR_PIMM64) {
| LOAD_32BIT_VAL tmp_reg2, Z_OFFSET(addr)
| ldr tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2]
| add_sub_ins tmp_reg1, tmp_reg1, #val
| str tmp_reg1, [Rx(Z_REG(addr)), tmp_reg2]
|| } else {
| ldr tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)]
| add_sub_ins tmp_reg1, tmp_reg1, #val
| str tmp_reg1, [Rx(Z_REG(addr)), #Z_OFFSET(addr)]
|| }
|| } else if (Z_MODE(addr) == IS_REG) {
| add_sub_ins Rx(Z_REG(addr)), Rx(Z_REG(addr)), #val
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
/* Compare value from memory 'addr' with immediate value 'val'.
* Note: the equivalent macro in JIT/x86 is LONG_OP_WITH_CONST. */
|.macro LONG_CMP_WITH_CONST, addr, val, tmp_reg1, tmp_reg2
|| if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, tmp_reg1, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg2
| CMP_64_WITH_CONST tmp_reg1, val, tmp_reg2
|| } else if (Z_MODE(addr) == IS_REG) {
| CMP_64_WITH_CONST Rx(Z_REG(addr)), val, tmp_reg1
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro GET_ZVAL_LVAL, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
|| if (Z_LVAL_P(Z_ZV(addr)) == 0) {
| mov Rx(reg), xzr
|| } else {
| LOAD_64BIT_VAL Rx(reg), Z_LVAL_P(Z_ZV(addr))
|| }
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, Rx(reg), Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|| } else if (Z_MODE(addr) == IS_REG) {
|| if (reg != Z_REG(addr)) {
| mov Rx(reg), Rx(Z_REG(addr))
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_MATH, opcode, reg, addr, tmp_reg1
|| switch (opcode) {
|| case ZEND_ADD:
| LONG_ADD_SUB adds, reg, addr, tmp_reg1
|| break;
|| case ZEND_SUB:
| LONG_ADD_SUB subs, reg, addr, tmp_reg1
|| break;
|| case ZEND_BW_OR:
| LONG_BW_OP orr, reg, addr, tmp_reg1
|| break;
|| case ZEND_BW_AND:
| LONG_BW_OP and, reg, addr, tmp_reg1
|| break;
|| case ZEND_BW_XOR:
| LONG_BW_OP eor, reg, addr, tmp_reg1
|| break;
|| default:
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_MATH_REG, opcode, dst_reg, src_reg1, src_reg2
|| switch (opcode) {
|| case ZEND_ADD:
| adds dst_reg, src_reg1, src_reg2
|| break;
|| case ZEND_SUB:
| subs dst_reg, src_reg1, src_reg2
|| break;
|| case ZEND_BW_OR:
| orr dst_reg, src_reg1, src_reg2
|| break;
|| case ZEND_BW_AND:
| and dst_reg, src_reg1, src_reg2
|| break;
|| case ZEND_BW_XOR:
| eor dst_reg, src_reg1, src_reg2
|| break;
|| default:
|| ZEND_UNREACHABLE();
|| }
|.endmacro
/* Store LONG value into memory 'addr'.
* This LONG value can be an immediate value i.e. 'val' in macro SET_ZVAL_LVAL, or
* a register value i.e. 'reg' in macro SET_ZVAL_LVAL_FROM_REG. */
|.macro SET_ZVAL_LVAL_FROM_REG, addr, reg, tmp_reg
|| if (Z_MODE(addr) == IS_REG) {
| mov Rx(Z_REG(addr)), reg
|| } else {
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_64_WITH_UOFFSET str, reg, Rx(Z_REG(addr)), Z_OFFSET(addr), tmp_reg
|| }
|.endmacro
|.macro SET_ZVAL_LVAL, addr, val, tmp_reg1, tmp_reg2
|| if (val == 0) {
| SET_ZVAL_LVAL_FROM_REG addr, xzr, tmp_reg2
|| } else {
| LOAD_64BIT_VAL tmp_reg1, val
| SET_ZVAL_LVAL_FROM_REG addr, tmp_reg1, tmp_reg2
|| }
|.endmacro
/* Store DOUBLE value from FP register 'reg' into memory 'addr'.
* Note: the equivalent macro in JIT/x86 is SSE_SET_ZVAL_DVAL. */
|.macro SET_ZVAL_DVAL, addr, reg, tmp_reg
|| if (Z_MODE(addr) == IS_REG) {
|| if (reg != Z_REG(addr)) {
| fmov Rd(Z_REG(addr)-ZREG_V0), Rd(reg-ZREG_V0)
|| }
|| } else {
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_64_WITH_UOFFSET str, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg)
|| }
|.endmacro
/* Load DOUBLE value from memory 'addr' into FP register 'reg'.
* Note: the equivalent macro in JIT/x86 is SSE_GET_ZVAL_DVAL. */
|.macro GET_ZVAL_DVAL, reg, addr, tmp_reg
|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) {
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| MEM_LOAD ldr, Rd(reg-ZREG_V0), Z_ZV(addr), Rx(tmp_reg)
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, Rd(reg-ZREG_V0), Rx(Z_REG(addr)), Z_OFFSET(addr), Rx(tmp_reg)
|| } else if (Z_MODE(addr) == IS_REG) {
| fmov Rd(reg-ZREG_V0), Rd(Z_REG(addr)-ZREG_V0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|| }
|.endmacro
|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg
|| if (Z_TYPE_P(zv) > IS_TRUE) {
|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg;
| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1)
| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2
|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : fp_tmp_reg;
| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), tmp_reg1
| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2
|| } else {
| // In x64, if the range of this LONG value can be represented via INT type, only move the low 32 bits into dst_addr.
| // Note that imm32 is signed extended to 64 bits during mov.
| // In aarch64, we choose to handle both cases in the same way. Even though 4 mov's are used for 64-bit value and 2 mov's are
| // needed for 32-bit value, an extra ext insn is needed for 32-bit value.
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
|| }
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2)
|| }
|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2)
|| }
|| }
|.endmacro
|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg1, tmp_reg2, fp_tmp_reg
|| if (Z_TYPE_P(zv) > IS_TRUE) {
|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ?
|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : fp_tmp_reg);
| MEM_LOAD ldr, Rd(dst_reg-ZREG_V0), zv, Rx(tmp_reg1)
| SET_ZVAL_DVAL dst_addr, dst_reg, tmp_reg2
| SET_ZVAL_DVAL res_addr, dst_reg, tmp_reg2
|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), tmp_reg1
| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg2
|| } else if (Z_MODE(res_addr) == IS_REG) {
| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), tmp_reg1
| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg2
|| } else {
| DOUBLE_GET_LONG fp_tmp_reg, Z_LVAL_P(zv), tmp_reg1
| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg2
| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg2
|| }
|| } else {
|| if (Z_MODE(dst_addr) == IS_REG) {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg1)
|| } else if (Z_MODE(res_addr) == IS_REG) {
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg1)
|| } else {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv), Rx(tmp_reg1), Rx(tmp_reg2)
|| }
|| }
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2)
|| }
|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2)
|| }
|| }
|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, Rw(tmp_reg1), Rx(tmp_reg2)
|| } else {
| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv), Rw(tmp_reg1), Rx(tmp_reg2)
|| }
|| }
|.endmacro
// the same as above, but "src" may overlap with "reg1"
// Useful info would be stored into reg1 and reg2, and they might be used afterward.
|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, tmp_reg2, fp_tmp_reg
| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg1, fp_tmp_reg
|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
|| !(src_info & MAY_BE_GUARD) &&
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
|| uint32_t type = concrete_type(src_info);
| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg1), Rx(tmp_reg2)
|| }
|| }
|| } else {
| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg1)
| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg1)
|| }
|.endmacro
|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, reg1, reg2, tmp_reg, fp_tmp_reg
|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
|| if (Z_MODE(src_addr) == IS_REG) {
|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg)
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg)
|| } else {
| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg)
| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg)
|| }
|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
|| if (Z_MODE(src_addr) == IS_REG) {
| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg
|| } else {
| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg
| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg
|| }
|| // Combine the following two branches.
|| // } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
|| } else {
| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg)
| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg)
|| }
|| }
|.endmacro
|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, reg1, reg2, tmp_reg, tmp_reg2, fp_tmp_reg
|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) {
|| if (Z_MODE(src_addr) == IS_REG) {
|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg)
|| }
|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(src_addr)), Rx(tmp_reg)
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr, Rx(tmp_reg)
|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) {
| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(Z_REG(dst_addr)), Rx(tmp_reg)
|| }
|| } else if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(res_addr), src_addr, Rx(tmp_reg)
| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(Z_REG(res_addr)), Rx(tmp_reg)
|| } else {
| GET_ZVAL_LVAL reg2, src_addr, Rx(tmp_reg)
| SET_ZVAL_LVAL_FROM_REG dst_addr, Rx(reg2), Rx(tmp_reg)
| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(reg2), Rx(tmp_reg)
|| }
|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
|| if (Z_MODE(src_addr) == IS_REG) {
| SET_ZVAL_DVAL dst_addr, Z_REG(src_addr), tmp_reg
| SET_ZVAL_DVAL res_addr, Z_REG(src_addr), tmp_reg
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_DVAL Z_REG(dst_addr), src_addr, tmp_reg
| SET_ZVAL_DVAL res_addr, Z_REG(dst_addr), tmp_reg
|| } else if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_DVAL Z_REG(res_addr), src_addr, tmp_reg
| SET_ZVAL_DVAL dst_addr, Z_REG(res_addr), tmp_reg
|| } else {
| GET_ZVAL_DVAL fp_tmp_reg, src_addr, tmp_reg
| SET_ZVAL_DVAL dst_addr, fp_tmp_reg, tmp_reg
| SET_ZVAL_DVAL res_addr, fp_tmp_reg, tmp_reg
|| }
|| } else {
| GET_ZVAL_PTR Rx(reg2), src_addr, Rx(tmp_reg)
| SET_ZVAL_PTR dst_addr, Rx(reg2), Rx(tmp_reg)
| SET_ZVAL_PTR res_addr, Rx(reg2), Rx(tmp_reg)
|| }
|| }
|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| uint32_t type = concrete_type(src_info);
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
| SET_ZVAL_TYPE_INFO dst_addr, type, Rw(tmp_reg), Rx(tmp_reg2)
|| }
|| }
|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, type, Rw(tmp_reg), Rx(tmp_reg2)
|| }
|| } else {
| GET_ZVAL_TYPE_INFO Rw(reg1), src_addr, Rx(tmp_reg)
| SET_ZVAL_TYPE_INFO_FROM_REG dst_addr, Rw(reg1), Rx(tmp_reg)
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, Rw(reg1), Rx(tmp_reg)
|| }
|.endmacro
|.macro IF_UNDEF, type_reg, label
| cbz type_reg, label
|.endmacro
|.macro IF_TYPE, type, val, label
|| if (val == 0) {
| cbz type, label
|| } else {
| cmp type, #val
| beq label
|| }
|.endmacro
|.macro IF_NOT_TYPE, type, val, label
|| if (val == 0) {
| cbnz type, label
|| } else {
| cmp type, #val
| bne label
|| }
|.endmacro
|.macro IF_Z_TYPE, zv, val, label, tmp_reg
| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)]
| IF_TYPE tmp_reg, val, label
|.endmacro
|.macro IF_NOT_Z_TYPE, zv, val, label, tmp_reg
| ldrb tmp_reg, [zv, #offsetof(zval, u1.v.type)]
| IF_NOT_TYPE tmp_reg, val, label
|.endmacro
|.macro CMP_ZVAL_TYPE, addr, val, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg)
| cmp Rw(tmp_reg), #val
|.endmacro
|.macro IF_ZVAL_TYPE, addr, val, label, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg)
| IF_TYPE Rw(tmp_reg), val, label
|.endmacro
|.macro IF_NOT_ZVAL_TYPE, addr, val, label, tmp_reg
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type), Rx(tmp_reg)
| IF_NOT_TYPE Rw(tmp_reg), val, label
|.endmacro
|.macro IF_FLAGS, type_flags, mask, label, tmp_reg
| TST_32_WITH_CONST type_flags, mask, tmp_reg
| bne label
|.endmacro
|.macro IF_NOT_FLAGS, type_flags, mask, label, tmp_reg
| TST_32_WITH_CONST type_flags, mask, tmp_reg
| beq label
|.endmacro
|.macro IF_REFCOUNTED, type_flags, label, tmp_reg
| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg
| bne label
|.endmacro
|.macro IF_NOT_REFCOUNTED, type_flags, label, tmp_reg
| TST_32_WITH_CONST type_flags, (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), tmp_reg
| beq label
|.endmacro
|.macro IF_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2)
| IF_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2)
|.endmacro
|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label, tmp_reg1, tmp_reg2
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| MEM_ACCESS_8_WITH_UOFFSET ldrb, Rw(tmp_reg1), Rx(Z_REG(addr)), Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags), Rx(tmp_reg2)
| IF_NOT_FLAGS Rw(tmp_reg1), mask, label, Rw(tmp_reg2)
|.endmacro
|.macro IF_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2
| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2
|.endmacro
|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label, tmp_reg1, tmp_reg2
| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label, tmp_reg1, tmp_reg2
|.endmacro
|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label, tmp_reg1, tmp_reg2
| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label, tmp_reg1, tmp_reg2
|.endmacro
|.macro GC_ADDREF, zv, tmp_reg
| ldr tmp_reg, [zv]
| add tmp_reg, tmp_reg, #1
| str tmp_reg, [zv]
|.endmacro
|.macro GC_ADDREF_2, zv, tmp_reg
| ldr tmp_reg, [zv]
| add tmp_reg, tmp_reg, #2
| str tmp_reg, [zv]
|.endmacro
|.macro GC_DELREF, zv, tmp_reg
| ldr tmp_reg, [zv]
| subs tmp_reg, tmp_reg, #1
| str tmp_reg, [zv]
|.endmacro
|.macro IF_GC_MAY_NOT_LEAK, ptr, label, tmp_reg1, tmp_reg2
| ldr tmp_reg1, [ptr, #4]
| TST_32_WITH_CONST tmp_reg1, (GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)), tmp_reg2
| bne label
|.endmacro
|.macro ADDREF_CONST, zv, tmp_reg1, tmp_reg2
| LOAD_ADDR tmp_reg1, Z_PTR_P(zv)
| ldr tmp_reg2, [tmp_reg1]
| add tmp_reg2, tmp_reg2, #1
| str tmp_reg2, [tmp_reg1]
|.endmacro
|.macro ADDREF_CONST_2, zv, tmp_reg1, tmp_reg2
| LOAD_ADDR tmp_reg1, Z_PTR_P(zv)
| ldr tmp_reg2, [tmp_reg1]
| add tmp_reg2, tmp_reg2, #2
| str tmp_reg2, [tmp_reg1]
|.endmacro
|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg, tmp_reg
|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg
|| }
| GC_ADDREF value_ptr_reg, tmp_reg
|1:
|| }
|.endmacro
|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg, tmp_reg
|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_REFCOUNTED type_flags_reg, >1, tmp_reg
|| }
| ldr tmp_reg, [value_ptr_reg]
| add tmp_reg, tmp_reg, #2
| str tmp_reg, [value_ptr_reg]
|1:
|| }
|.endmacro
|.macro ZVAL_DEREF, reg, info, tmp_reg
|| if (info & MAY_BE_REF) {
| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1, tmp_reg
| GET_Z_PTR reg, reg
| add reg, reg, #offsetof(zend_reference, val)
|1:
|| }
|.endmacro
|.macro SET_EX_OPLINE, op, tmp_reg
|| if (op == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| SAVE_IP
|| } else {
| ADDR_STORE EX->opline, op, tmp_reg
|| if (!GCC_GLOBAL_REGS) {
|| zend_jit_reset_last_valid_opline();
|| }
|| }
|.endmacro
// arg1 "zval" should be in FCARG1x
|.macro ZVAL_DTOR_FUNC, var_info, opline, tmp_reg
|| do {
|| if (!((var_info) & MAY_BE_GUARD)
|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|| uint8_t type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
|| if (type == IS_STRING && !ZEND_DEBUG) {
| EXT_CALL _efree, tmp_reg
|| break;
|| } else if (type == IS_ARRAY) {
|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
| SET_EX_OPLINE opline, tmp_reg
|| }
| EXT_CALL zend_array_destroy, tmp_reg
|| } else {
| EXT_CALL zend_jit_array_free, tmp_reg
|| }
|| break;
|| } else if (type == IS_OBJECT) {
|| if (opline) {
| SET_EX_OPLINE opline, REG0
|| }
| EXT_CALL zend_objects_store_del, tmp_reg
|| break;
|| }
|| }
|| if (opline) {
| SET_EX_OPLINE opline, tmp_reg
|| }
| EXT_CALL rc_dtor_func, tmp_reg
|| } while(0);
|.endmacro
|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline, tmp_reg1, tmp_reg2
|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
|| if (cold) {
| IF_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2
|.cold_code
|1:
|| } else {
| IF_NOT_ZVAL_REFCOUNTED addr, >4, tmp_reg1, tmp_reg2
|| }
|| }
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1x, addr, Rx(tmp_reg2)
| GC_DELREF FCARG1x, Rw(tmp_reg1)
|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
| bne >3
|| } else {
| bne >4
|| }
|| }
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op_info, opline, Rx(tmp_reg1)
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
| b >4
|| }
|3:
|| }
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val));
| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1, tmp_reg1
| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4, tmp_reg1, tmp_reg2
| GET_ZVAL_PTR FCARG1x, ref_addr, Rx(tmp_reg2)
|1:
|| }
| IF_GC_MAY_NOT_LEAK FCARG1x, >4, Rw(tmp_reg1), Rw(tmp_reg2)
| // gc_possible_root(Z_COUNTED_P(z))
| EXT_CALL gc_possible_root, Rx(tmp_reg1)
|| }
|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) {
| b >4
|.code
|| }
|4:
|| }
|.endmacro
|.macro FREE_OP, op_type, op, op_info, cold, opline, tmp_reg1, tmp_reg2
|| if (op_type & (IS_VAR|IS_TMP_VAR)) {
|| zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
| ZVAL_PTR_DTOR addr, op_info, 0, cold, opline, tmp_reg1, tmp_reg2
|| }
|.endmacro
|.macro SEPARATE_ARRAY, addr, op_info, cold, tmp_reg1, tmp_reg2
|| if (RC_MAY_BE_N(op_info)) {
|| if (Z_REG(addr) != ZREG_FP) {
| GET_ZVAL_LVAL ZREG_REG0, addr, Rx(tmp_reg1)
|| if (RC_MAY_BE_1(op_info)) {
| // if (GC_REFCOUNT() > 1)
| ldr Rw(tmp_reg1), [REG0]
| cmp Rw(tmp_reg1), #1
| bls >2
|| }
|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, addr
|| }
| EXT_CALL zend_jit_zval_array_dup, REG0
| mov REG0, RETVALx
|2:
| mov FCARG1x, REG0
|| } else {
| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1)
|| if (RC_MAY_BE_1(op_info)) {
| // if (GC_REFCOUNT() > 1)
| ldr Rw(tmp_reg1), [FCARG1x]
| cmp Rw(tmp_reg1), #1
|| if (cold) {
| bhi >1
|.cold_code
|1:
|| } else {
| bls >2
|| }
|| }
| IF_NOT_ZVAL_REFCOUNTED addr, >1, tmp_reg1, tmp_reg2
| GC_DELREF FCARG1x, Rw(tmp_reg1)
|1:
| EXT_CALL zend_array_dup, REG0
| mov REG0, RETVALx
| SET_ZVAL_PTR addr, REG0, Rx(tmp_reg1)
| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX, Rw(tmp_reg1), Rx(tmp_reg2)
| mov FCARG1x, REG0
|| if (RC_MAY_BE_1(op_info)) {
|| if (cold) {
| b >2
|.code
|| }
|| }
|2:
|| }
|| } else {
| GET_ZVAL_LVAL ZREG_FCARG1, addr, Rx(tmp_reg1)
|| }
|.endmacro
/* argument is passed in FCARG1x */
|.macro EFREE_REFERENCE
||#if ZEND_DEBUG
| mov FCARG2x, xzr // filename
| mov CARG3w, wzr // lineno
| mov CARG4, xzr
| mov CARG5, xzr
| EXT_CALL _efree, REG0
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
| EXT_CALL _efree_32, REG0
||#else
| EXT_CALL _efree, REG0
||#endif
||#endif
|.endmacro
|.macro EMALLOC, size, op_array, opline
||#if ZEND_DEBUG
|| const char *filename = op_array->filename ? op_array->filename->val : NULL;
| mov FCARG1x, #size
| LOAD_ADDR FCARG2x, filename
| LOAD_32BIT_VAL CARG3w, opline->lineno
| mov CARG4, xzr
| mov CARG5, xzr
| EXT_CALL _emalloc, REG0
| mov REG0, RETVALx
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
|| if (size > 24 && size <= 32) {
| EXT_CALL _emalloc_32, REG0
| mov REG0, RETVALx
|| } else {
| mov FCARG1x, #size
| EXT_CALL _emalloc, REG0
| mov REG0, RETVALx
|| }
||#else
| mov FCARG1x, #size
| EXT_CALL _emalloc, REG0
| mov REG0, RETVALx
||#endif
||#endif
|.endmacro
|.macro OBJ_RELEASE, reg, exit_label, tmp_reg1, tmp_reg2
| GC_DELREF Rx(reg), Rw(tmp_reg1)
| bne >1
| // zend_objects_store_del(obj);
|| if (reg != ZREG_FCARG1) {
| mov FCARG1x, Rx(reg)
|| }
| EXT_CALL zend_objects_store_del, Rx(tmp_reg1)
| b exit_label
|1:
| IF_GC_MAY_NOT_LEAK Rx(reg), >1, Rw(tmp_reg1), Rw(tmp_reg2)
| // gc_possible_root(obj)
|| if (reg != ZREG_FCARG1) {
| mov FCARG1x, Rx(reg)
|| }
| EXT_CALL gc_possible_root, Rx(tmp_reg1)
|1:
|.endmacro
|.macro UNDEFINED_OFFSET, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| bl ->undefined_offset_ex
|| } else {
| SET_EX_OPLINE opline, REG0
| bl ->undefined_offset
|| }
|.endmacro
|.macro UNDEFINED_INDEX, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| bl ->undefined_index_ex
|| } else {
| SET_EX_OPLINE opline, REG0
| bl ->undefined_index
|| }
|.endmacro
|.macro CANNOT_ADD_ELEMENT, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| bl ->cannot_add_element_ex
|| } else {
| SET_EX_OPLINE opline, REG0
| bl ->cannot_add_element
|| }
|.endmacro
static bool reuse_ip = 0;
static bool delayed_call_chain = 0;
static uint32_t delayed_call_level = 0;
static const zend_op *last_valid_opline = NULL;
static bool use_last_vald_opline = 0;
static bool track_last_valid_opline = 0;
static int jit_return_label = -1;
static uint32_t current_trace_num = 0;
static uint32_t allowed_opt_flags = 0;
static void zend_jit_track_last_valid_opline(void)
{
use_last_vald_opline = 0;
track_last_valid_opline = 1;
}
static void zend_jit_use_last_valid_opline(void)
{
if (track_last_valid_opline) {
use_last_vald_opline = 1;
track_last_valid_opline = 0;
}
}
static bool zend_jit_trace_uses_initial_ip(void)
{
return use_last_vald_opline;
}
static void zend_jit_set_last_valid_opline(const zend_op *target_opline)
{
if (!reuse_ip) {
track_last_valid_opline = 0;
last_valid_opline = target_opline;
}
}
static void zend_jit_reset_last_valid_opline(void)
{
track_last_valid_opline = 0;
last_valid_opline = NULL;
}
static void zend_jit_start_reuse_ip(void)
{
zend_jit_reset_last_valid_opline();
reuse_ip = 1;
}
static int zend_jit_reuse_ip(dasm_State **Dst)
{
if (!reuse_ip) {
zend_jit_start_reuse_ip();
| // call = EX(call);
| ldr RX, EX->call
}
return 1;
}
static void zend_jit_stop_reuse_ip(void)
{
reuse_ip = 0;
}
static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
{
|->interrupt_handler:
| SAVE_IP
| //EG(vm_interrupt) = 0;
| MEM_STORE_8_ZTS strb, wzr, executor_globals, vm_interrupt, TMP1
| //if (EG(timed_out)) {
| MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, timed_out, TMP1
| cbz TMP1w, >1
| //zend_timeout();
| EXT_CALL zend_timeout, TMP1
|1:
| //} else if (zend_interrupt_function) {
if (zend_interrupt_function) {
| //zend_interrupt_function(execute_data);
| mov CARG1, FP
| EXT_CALL zend_interrupt_function, TMP1
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbz REG0, >1
| EXT_CALL zend_jit_exception_in_interrupt_handler_helper, TMP1
|1:
| //ZEND_VM_ENTER();
| //execute_data = EG(current_execute_data);
| MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1
| LOAD_IP
}
| //ZEND_VM_CONTINUE()
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP TMP1
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
| JMP_IP TMP1
} else {
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| mov RETVALx, #1 // ZEND_VM_ENTER
| ret
}
return 1;
}
static int zend_jit_exception_handler_stub(dasm_State **Dst)
{
|->exception_handler:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
const void *handler = zend_get_opcode_handler_func(EG(exception_op));
| ADD_HYBRID_SPAD
| EXT_CALL handler, REG0
| JMP_IP TMP1
} else {
const void *handler = EG(exception_op)->handler;
if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
| EXT_JMP handler, REG0
} else {
| mov FCARG1x, FP
| EXT_CALL handler, REG0
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| tbnz RETVALw, #31, >1
| mov RETVALw, #1 // ZEND_VM_ENTER
|1:
| ret
}
}
return 1;
}
static int zend_jit_exception_handler_undef_stub(dasm_State **Dst)
{
|->exception_handler_undef:
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, opline_before_exception, REG0
| ldrb TMP1w, OP:REG0->result_type
| TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
| beq >1
| ldr REG0w, OP:REG0->result.var
| add REG0, REG0, FP
| SET_Z_TYPE_INFO REG0, IS_UNDEF, TMP1w
|1:
| b ->exception_handler
return 1;
}
static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst)
{
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
|->exception_handler_free_op1_op2:
| UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w
| ldrb TMP1w, OP:RX->op1_type
| TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
| beq >9
| ldr REG0w, OP:RX->op1.var
| add REG0, REG0, FP
| ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
|9:
| ldrb TMP1w, OP:RX->op2_type
| TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
| beq >9
| ldr REG0w, OP:RX->op2.var
| add REG0, REG0, FP
| ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
|9:
| b ->exception_handler
return 1;
}
static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst)
{
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
|->exception_handler_free_op2:
| MEM_LOAD_64_ZTS ldr, RX, executor_globals, opline_before_exception, REG0
| UNDEF_OPLINE_RESULT_IF_USED TMP1w, TMP2w
| ldrb TMP1w, OP:RX->op2_type
| TST_32_WITH_CONST TMP1w, (IS_TMP_VAR|IS_VAR), TMP2w
| beq >9
| ldr REG0w, OP:RX->op2.var
| add REG0, REG0, FP
| ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
|9:
| b ->exception_handler
return 1;
}
static int zend_jit_leave_function_stub(dasm_State **Dst)
{
|->leave_function_handler:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w
| bne >1
| EXT_CALL zend_jit_leave_nested_func_helper, REG0
| ADD_HYBRID_SPAD
| JMP_IP TMP1
|1:
| EXT_CALL zend_jit_leave_top_func_helper, REG0
| ADD_HYBRID_SPAD
| JMP_IP TMP1
} else {
if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
} else {
| mov FCARG2x, FP
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
}
| TST_32_WITH_CONST FCARG1w, ZEND_CALL_TOP, TMP1w
| bne >1
| EXT_JMP zend_jit_leave_nested_func_helper, REG0
|1:
| EXT_JMP zend_jit_leave_top_func_helper, REG0
}
return 1;
}
static int zend_jit_leave_throw_stub(dasm_State **Dst)
{
|->leave_throw_handler:
| // if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
if (GCC_GLOBAL_REGS) {
| ldrb TMP1w, OP:IP->opcode
| cmp TMP1w, #ZEND_HANDLE_EXCEPTION
| beq >5
| // EG(opline_before_exception) = opline;
| MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2
|5:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2
| str IP, EX->opline
| // HANDLE_EXCEPTION()
| b ->exception_handler
} else {
| GET_IP TMP1
| ldrb TMP2w, OP:TMP1->opcode
| cmp TMP2w, #ZEND_HANDLE_EXCEPTION
| beq >5
| // EG(opline_before_exception) = opline;
| MEM_STORE_64_ZTS str, TMP1, executor_globals, opline_before_exception, TMP2
|5:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| mov RETVALx, #2 // ZEND_VM_LEAVE
| ret
}
return 1;
}
static int zend_jit_icall_throw_stub(dasm_State **Dst)
{
|->icall_throw_handler:
| // zend_rethrow_exception(zend_execute_data *execute_data)
| ldr IP, EX->opline
| // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
| ldrb TMP1w, OP:IP->opcode
| cmp TMP1w, #ZEND_HANDLE_EXCEPTION
| beq >1
| // EG(opline_before_exception) = opline;
| MEM_STORE_64_ZTS str, IP, executor_globals, opline_before_exception, TMP2
|1:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op, TMP2
|| if (GCC_GLOBAL_REGS) {
| str IP, EX->opline
|| }
| // HANDLE_EXCEPTION()
| b ->exception_handler
return 1;
}
static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst)
{
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
|->throw_cannot_pass_by_ref:
| ldr REG0, EX->opline
| ldr REG1w, OP:REG0->result.var
| add REG1, REG1, RX
| SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w
| // last EX(call) frame may be delayed
| ldr TMP1, EX->call
| cmp RX, TMP1
| beq >1
| ldr REG1, EX->call
| str REG1, EX:RX->prev_execute_data
| str RX, EX->call
|1:
| mov RX, REG0
| ldr FCARG1w, OP:REG0->op2.num
| EXT_CALL zend_cannot_pass_by_reference, REG0
| ldrb TMP1w, OP:RX->op1_type
| cmp TMP1w, #IS_TMP_VAR
| bne >9
| ldr REG0w, OP:RX->op1.var
| add REG0, REG0, FP
| ZVAL_PTR_DTOR addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL, ZREG_TMP1, ZREG_TMP2
|9:
| b ->exception_handler
return 1;
}
static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst)
{
|->undefined_offset_ex:
| SAVE_IP
| b ->undefined_offset
return 1;
}
static int zend_jit_undefined_offset_stub(dasm_State **Dst)
{
|->undefined_offset:
|| if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, FP
|| }
| EXT_JMP zend_jit_undefined_long_key, REG0
return 1;
}
static int zend_jit_undefined_index_ex_stub(dasm_State **Dst)
{
|->undefined_index_ex:
| SAVE_IP
| b ->undefined_index
return 1;
}
static int zend_jit_undefined_index_stub(dasm_State **Dst)
{
|->undefined_index:
|| if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, FP
|| }
| EXT_JMP zend_jit_undefined_string_key, REG0
return 1;
}
static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst)
{
|->cannot_add_element_ex:
| SAVE_IP
| b ->cannot_add_element
return 1;
}
static int zend_jit_cannot_add_element_stub(dasm_State **Dst)
{
|->cannot_add_element:
| // sub r4, 8
| ldr REG0, EX->opline
| ldrb TMP1w, OP:REG0->result_type
| cmp TMP1w, #IS_UNUSED
| beq >1
| ldr REG0w, OP:REG0->result.var
| add REG0, REG0, FP
| SET_Z_TYPE_INFO REG0, IS_NULL, TMP1w
|1:
| mov CARG1, xzr
| LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
| EXT_JMP zend_throw_error, REG0 // tail call
| // add r4, 8
| //ret
return 1;
}
static int zend_jit_undefined_function_stub(dasm_State **Dst)
{
|->undefined_function:
| ldr REG0, EX->opline
| mov CARG1, xzr
| LOAD_ADDR CARG2, "Call to undefined function %s()"
| ldrsw CARG3, [REG0, #offsetof(zend_op, op2.constant)]
| ldr CARG3, [REG0, CARG3]
| add CARG3, CARG3, #offsetof(zend_string, val)
#ifdef __APPLE__
| str CARG3, [sp, #-16]!
#endif
| EXT_CALL zend_throw_error, REG0
#ifdef __APPLE__
| add sp, sp, #16
#endif
| b ->exception_handler
return 1;
}
static int zend_jit_negative_shift_stub(dasm_State **Dst)
{
|->negative_shift:
| ldr RX, EX->opline
| LOAD_ADDR CARG1, zend_ce_arithmetic_error
| LOAD_ADDR CARG2, "Bit shift by negative number"
| EXT_CALL zend_throw_error, REG0
| b ->exception_handler_free_op1_op2
return 1;
}
static int zend_jit_mod_by_zero_stub(dasm_State **Dst)
{
|->mod_by_zero:
| ldr RX, EX->opline
| LOAD_ADDR CARG1, zend_ce_division_by_zero_error
| LOAD_ADDR CARG2, "Modulo by zero"
| EXT_CALL zend_throw_error, REG0
| b ->exception_handler_free_op1_op2
return 1;
}
static int zend_jit_invalid_this_stub(dasm_State **Dst)
{
|->invalid_this:
| UNDEF_OPLINE_RESULT TMP1w
| mov CARG1, xzr
| LOAD_ADDR CARG2, "Using $this when not in object context"
| EXT_CALL zend_throw_error, REG0
| b ->exception_handler
return 1;
}
static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_runtime_jit:
| EXT_CALL zend_runtime_jit, REG0
| JMP_IP TMP1
return 1;
}
static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_profile_jit:
| // ++zend_jit_profile_counter;
| LOAD_ADDR REG0, &zend_jit_profile_counter
| ldr TMP1, [REG0]
| add TMP1, TMP1, #1
| str TMP1, [REG0]
| // op_array = (zend_op_array*)EX(func);
| ldr REG0, EX->func
| // run_time_cache = EX(run_time_cache);
| ldr REG2, EX->run_time_cache
| // jit_extension = (const void*)ZEND_FUNC_INFO(op_array);
| ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| // ++ZEND_COUNTER_INFO(op_array)
|| if ((zend_jit_profile_counter_rid * sizeof(void*)) > LDR_STR_PIMM64) {
| LOAD_32BIT_VAL TMP1, (zend_jit_profile_counter_rid * sizeof(void*))
| ldr TMP2, [REG2, TMP1]
| add TMP2, TMP2, #1
| str TMP2, [REG2, TMP1]
|| } else {
| ldr TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))]
| add TMP2, TMP2, #1
| str TMP2, [REG2, #(zend_jit_profile_counter_rid * sizeof(void*))]
|| }
| // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)()
| ldr TMP1, [REG0, #offsetof(zend_jit_op_array_extension, orig_handler)]
| br TMP1
return 1;
}
static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_hot_code:
|| ZEND_ASSERT(ZEND_JIT_COUNTER_INIT <= MOVZ_IMM);
| movz TMP1w, #ZEND_JIT_COUNTER_INIT
| strh TMP1w, [REG2]
| mov FCARG1x, FP
| GET_IP FCARG2x
| EXT_CALL zend_jit_hot_func, REG0
| JMP_IP TMP1
return 1;
}
/*
* This code is based Mike Pall's "Hashed profile counters" idea, implemented
* in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual
* property disclosure and research opportunities" email
* at http://lua-users.org/lists/lua-l/2009-11/msg00089.html
*
* In addition we use a variation of Knuth's multiplicative hash function
* described at https://code.i-harness.com/en/q/a21ce
*
* uint64_t hash(uint64_t x) {
* x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
* x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
* x = x ^ (x >> 31);
* return x;
* }
*
* uint_32_t hash(uint32_t x) {
* x = ((x >> 16) ^ x) * 0x45d9f3b;
* x = ((x >> 16) ^ x) * 0x45d9f3b;
* x = (x >> 16) ^ x;
* return x;
* }
*
*/
static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost)
{
| ldr REG0, EX->func
| ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG2, [REG1, #offsetof(zend_jit_op_array_hot_extension, counter)]
| ldrh TMP2w, [REG2]
| ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w
| strh TMP2w, [REG2]
| ble ->hybrid_hot_code
| GET_IP REG2
| ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)]
| sub REG2, REG2, TMP1
| // divide by sizeof(zend_op)
|| ZEND_ASSERT(sizeof(zend_op) == 32);
| add TMP1, REG1, REG2, asr #2
| ldr TMP1, [TMP1, #offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
| br TMP1
return 1;
}
static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 1;
}
|->hybrid_func_hot_counter:
return zend_jit_hybrid_hot_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 1;
}
|->hybrid_loop_hot_counter:
return zend_jit_hybrid_hot_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
// On entry from counter stub:
// REG2 -> zend_op_trace_info.counter
|->hybrid_hot_trace:
| mov TMP1w, #ZEND_JIT_COUNTER_INIT
| strh TMP1w, [REG2]
| mov FCARG1x, FP
| GET_IP FCARG2x
| EXT_CALL zend_jit_trace_hot_root, REG0
| tbnz RETVALw, #31, >1 // Result is < 0 on failure.
| MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0
| LOAD_IP
| JMP_IP TMP1
|1:
| EXT_JMP zend_jit_halt_op->handler, REG0
return 1;
}
static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost)
{
| ldr REG0, EX->func
| ldr REG1, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG1, [REG1, #offsetof(zend_jit_op_array_trace_extension, offset)]
| add TMP1, REG1, IP
| ldr REG2, [TMP1, #offsetof(zend_op_trace_info, counter)]
| ldrh TMP2w, [REG2]
| ADD_SUB_32_WITH_CONST subs, TMP2w, TMP2w, cost, TMP1w
| strh TMP2w, [REG2]
| ble ->hybrid_hot_trace
// Note: "REG1 + IP" is re-calculated as TMP1 is used as temporary register by the prior
// ADD_SUB_32_WITH_CONST. Will optimize in the future if more temporary registers are available.
| add TMP1, REG1, IP
| ldr TMP2, [TMP1, #offsetof(zend_op_trace_info, orig_handler)]
| br TMP2
return 1;
}
static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 1;
}
|->hybrid_func_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
return 1;
}
|->hybrid_ret_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
}
static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 1;
}
|->hybrid_loop_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_trace_halt_stub(dasm_State **Dst)
{
|->trace_halt:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| EXT_JMP zend_jit_halt_op->handler, REG0
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
| mov IP, xzr // PC must be zero
| ret
} else {
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| movn RETVALx, #0 // ZEND_VM_RETURN (-1)
| ret
}
return 1;
}
static int zend_jit_trace_exit_stub(dasm_State **Dst)
{
|->trace_exit:
|
| // Save CPU registers(32 GP regs + 32 FP regs) on stack in the order of d31 to x0
|
| stp d30, d31, [sp, #-16]!
| stp d28, d29, [sp, #-16]!
| stp d26, d27, [sp, #-16]!
| stp d24, d25, [sp, #-16]!
| stp d22, d23, [sp, #-16]!
| stp d20, d21, [sp, #-16]!
| stp d18, d19, [sp, #-16]!
| stp d16, d17, [sp, #-16]!
| //stp d14, d15, [sp, #-16]! // we don't use preserved registers yet
| //stp d12, d13, [sp, #-16]!
| //stp d10, d11, [sp, #-16]!
| //stp d8, d9, [sp, #-16]!
| stp d6, d7, [sp, #(-16*5)]!
| stp d4, d5, [sp, #-16]!
| stp d2, d3, [sp, #-16]!
| stp d0, d1, [sp, #-16]!
|
| //str x30, [sp, #-16]! // we don't use callee-saved registers yet (x31 can be omitted)
| stp x28, x29, [sp, #(-16*2)]! // we have to store RX (x28)
| //stp x26, x27, [sp, #-16]! // we don't use callee-saved registers yet
| //stp x24, x25, [sp, #-16]!
| //stp x22, x23, [sp, #-16]!
| //stp x20, x21, [sp, #-16]!
| //stp x18, x19, [sp, #-16]!
| //stp x16, x17, [sp, #-16]! // we don't need temporary registers
| stp x14, x15, [sp, #-(16*7)]!
| stp x12, x13, [sp, #-16]!
| stp x10, x11, [sp, #-16]!
| stp x8, x9, [sp, #-16]!
| stp x6, x7, [sp, #-16]!
| stp x4, x5, [sp, #-16]!
| stp x2, x3, [sp, #-16]!
| stp x0, x1, [sp, #-16]!
|
| mov FCARG1w, TMP1w // exit_num
| mov FCARG2x, sp
|
| // EX(opline) = opline
| SAVE_IP
| // zend_jit_trace_exit(trace_num, exit_num)
| EXT_CALL zend_jit_trace_exit, REG0
|
| add sp, sp, #(32 * 16) // restore sp
|
| tst RETVALw, RETVALw
| bne >1 // not zero
| // execute_data = EG(current_execute_data)
| MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0
| // opline = EX(opline)
| LOAD_IP
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP TMP1
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
| JMP_IP TMP1
} else {
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| mov RETVALx, #1 // ZEND_VM_ENTER
| ret
}
|1:
| blt ->trace_halt
| // execute_data = EG(current_execute_data)
| MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, REG0
| // opline = EX(opline)
| LOAD_IP
| // check for interrupt (try to avoid this ???)
| MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
| cbnz TMP1w, ->interrupt_handler
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| ldr REG0, EX->func
| ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
| ldr REG0, [IP, REG0]
| br REG0
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
| ldr REG0, EX->func
| ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
| ldr REG0, [IP, REG0]
| br REG0
} else {
| ldr IP, EX->opline
| mov FCARG1x, FP
| ldr REG0, EX->func
| ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
| ldr REG0, [IP, REG0]
| blr REG0
|
| tst RETVALw, RETVALw
| blt ->trace_halt
|
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| mov RETVALx, #1 // ZEND_VM_ENTER
| ret
}
return 1;
}
static int zend_jit_trace_escape_stub(dasm_State **Dst)
{
|->trace_escape:
|
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP, TMP1
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
| JMP_IP, TMP1
} else {
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| mov RETVALx, #1 // ZEND_VM_ENTER
| ret
}
return 1;
}
/* Keep 32 exit points in a single code block */
#define ZEND_JIT_EXIT_POINTS_SPACING 4 // bl = bytes
#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n)
{
uint32_t i;
| bl >2
|1:
for (i = 1; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) {
| bl >2
}
|2:
| adr TMP1, <1
| sub TMP1, lr, TMP1
| lsr TMP1, TMP1, #2
if (n) {
| ADD_SUB_32_WITH_CONST add, TMP1w, TMP1w, n, TMP2w
}
| b ->trace_exit // pass exit_num in TMP1w
return 1;
}
#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
{
|->context_threaded_call:
| NIY_STUB // TODO
return 1;
}
#endif
static int zend_jit_assign_const_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
|->assign_const:
| stp x29, x30, [sp, #-32]!
| mov x29, sp
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CONST, val_addr, val_info,
0, 0)) {
return 0;
}
| ldp x29, x30, [sp], #32
| ret
return 1;
}
static int zend_jit_assign_tmp_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
|->assign_tmp:
| stp x29, x30, [sp, #-32]!
| mov x29, sp
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_TMP_VAR, val_addr, val_info,
0, 0)) {
return 0;
}
| ldp x29, x30, [sp], #32
| ret
return 1;
}
static int zend_jit_assign_var_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
|->assign_var:
| stp x29, x30, [sp, #-32]!
| mov x29, sp
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_VAR, val_addr, val_info,
0, 0)) {
return 0;
}
| ldp x29, x30, [sp], #32
| ret
return 1;
}
static int zend_jit_assign_cv_noref_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
|->assign_cv_noref:
| stp x29, x30, [sp, #-32]!
| mov x29, sp
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0)) {
return 0;
}
| ldp x29, x30, [sp], #32
| ret
return 1;
}
static int zend_jit_assign_cv_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
|->assign_cv:
| stp x29, x30, [sp, #-32]!
| mov x29, sp
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0)) {
return 0;
}
| ldp x29, x30, [sp], #32
| ret
return 1;
}
static const zend_jit_stub zend_jit_stubs[] = {
JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN),
#ifdef CONTEXT_THREADED_JIT
JIT_STUB(context_threaded_call, SP_ADJ_NONE, SP_ADJ_NONE),
#endif
};
#ifdef HAVE_GDB
# if 0
typedef struct _Unwind_Context _Unwind_Context;
typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
typedef struct _zend_jit_unwind_arg {
int cnt;
uintptr_t cfa[3];
} zend_jit_unwind_arg;
static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
{
zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
arg->cnt++;
if (arg->cnt == 3) {
return 5; // _URC_END_OF_STACK
}
return 0; // _URC_NO_REASON;
}
static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
{
zend_jit_unwind_arg arg;
memset(&arg, 0, sizeof(arg));
_Unwind_Backtrace(zend_jit_unwind_cb, &arg);
if (arg.cnt == 3) {
sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1];
}
}
# else
static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
{
uintptr_t ret;
__asm__ (
"ldr %0, [x29]\n\t"
"sub %0 ,%0, x29"
: "=r"(ret));
sp_adj[SP_ADJ_VM] = ret;
}
# endif
extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);
static zend_never_inline void zend_jit_set_sp_adj_vm(void)
{
void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);
orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
execute_ex(NULL); // set sp_adj[SP_ADJ_VM]
zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
}
#endif
static int zend_jit_setup(void)
{
allowed_opt_flags = 0;
#if ZTS
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
# if defined(__FreeBSD__)
if (tsrm_ls_cache_tcb_offset == 0) {
TLSDescriptor **where;
__asm__(
"adrp %0, :tlsdesc:_tsrm_ls_cache\n"
"add %0, %0, :tlsdesc_lo12:_tsrm_ls_cache\n"
: "=r" (where));
/* See https://github.com/ARM-software/abi-aa/blob/2a70c42d62e9c3eb5887fa50b71257f20daca6f9/aaelf64/aaelf64.rst
* section "Relocations for thread-local storage".
* The first entry holds a pointer to the variable's TLS descriptor resolver function and the second entry holds
* a platform-specific offset or pointer. */
TLSDescriptor *tlsdesc = where[1];
tsrm_tls_offset = tlsdesc->offset;
/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/22ca6db50f4e6bd75a141f57cf953d8de6531a06/lib/libc/gen/tls.c#L88) */
tsrm_tls_index = (tlsdesc->index + 1) * 8;
}
# else
ZEND_ASSERT(tsrm_ls_cache_tcb_offset != 0);
# endif
#endif
memset(sp_adj, 0, sizeof(sp_adj));
#ifdef HAVE_GDB
sp_adj[SP_ADJ_RET] = 0;
sp_adj[SP_ADJ_ASSIGN] = 32;
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM]
#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD
#else
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM];
#endif
} else if (GCC_GLOBAL_REGS) {
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD
} else {
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD
}
#endif
return SUCCESS;
}
static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst)
{
| brk #0
return 1;
}
static int zend_jit_align_func(dasm_State **Dst)
{
reuse_ip = 0;
delayed_call_chain = 0;
last_valid_opline = NULL;
use_last_vald_opline = 0;
track_last_valid_opline = 0;
jit_return_label = -1;
|.align 16
return 1;
}
static int zend_jit_align_stub(dasm_State **Dst)
{
|.align 16
return 1;
}
static int zend_jit_prologue(dasm_State **Dst)
{
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| SUB_HYBRID_SPAD
} else if (GCC_GLOBAL_REGS) {
| stp x29, x30, [sp, # -SPAD]! // stack alignment
|// mov x29, sp
} else {
| stp x29, x30, [sp, # -NR_SPAD]! // stack alignment
|// mov x29, sp
| stp FP, RX, T2 // save FP and IP
| mov FP, FCARG1x
}
return 1;
}
static int zend_jit_label(dasm_State **Dst, unsigned int label)
{
|=>label:
return 1;
}
static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
{
| // call->prev_execute_data = EX(call);
if (call_level == 1) {
| str xzr, EX:RX->prev_execute_data
} else {
| ldr REG0, EX->call
| str REG0, EX:RX->prev_execute_data
}
| // EX(call) = call;
| str RX, EX->call
delayed_call_chain = 0;
return 1;
}
static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
{
if (last_valid_opline == opline) {
zend_jit_use_last_valid_opline();
} else if (GCC_GLOBAL_REGS && last_valid_opline) {
zend_jit_use_last_valid_opline();
| LOAD_64BIT_VAL TMP1, (opline - last_valid_opline) * sizeof(zend_op)
| ADD_IP TMP1, TMP2
} else {
| LOAD_IP_ADDR opline
}
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg)
{
return zend_jit_set_ip(Dst, opline);
}
static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline)
{
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (!zend_jit_set_ip(Dst, opline)) {
return 0;
}
reuse_ip = 0;
return 1;
}
static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr)
{
| MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
if (exit_addr) {
| cbnz TMP1w, &exit_addr
} else if (last_valid_opline == opline) {
|| zend_jit_use_last_valid_opline();
| cbnz TMP1w, ->interrupt_handler
} else {
| cbnz TMP1w, >1
|.cold_code
|1:
| LOAD_IP_ADDR opline
| b ->interrupt_handler
|.code
}
return 1;
}
static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr)
{
if (timeout_exit_addr) {
| MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
| cbz TMP1w, =>loop_label
| b &timeout_exit_addr
} else {
| b =>loop_label
}
return 1;
}
static int zend_jit_check_exception(dasm_State **Dst)
{
| MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
| cbnz TMP2, ->exception_handler
return 1;
}
static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline)
{
if (opline->result_type & (IS_TMP_VAR|IS_VAR)) {
| MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
| cbnz TMP2, ->exception_handler_undef
return 1;
}
return zend_jit_check_exception(Dst);
}
static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num)
{
current_trace_num = trace_num;
| // EG(jit_trace_num) = trace_num;
| LOAD_32BIT_VAL TMP1w, trace_num
| MEM_STORE_32_ZTS str, TMP1w, executor_globals, jit_trace_num, TMP2
return 1;
}
static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t)
{
uint32_t i;
const void *exit_addr;
/* Emit veneers table for exit points (B instruction for each exit number) */
|.cold_code
for (i = 0; i < t->exit_count; i++) {
exit_addr = zend_jit_trace_get_exit_addr(i);
if (!exit_addr) {
return 0;
}
| b &exit_addr
}
|=>1: // end of the code
|.code
return 1;
}
static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr)
{
int ret = 0;
uint8_t *p, *end;
const void *veneer = NULL;
ptrdiff_t delta;
if (jmp_table_size) {
const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*)));
do {
if (*jmp_slot == from_addr) {
*jmp_slot = to_addr;
ret++;
}
jmp_slot++;
} while (--jmp_table_size);
}
end = (uint8_t*)code;
p = end + size;
while (p > end) {
uint32_t *ins_ptr;
uint32_t ins;
p -= 4;
ins_ptr = (uint32_t*)p;
ins = *ins_ptr;
if ((ins & 0xfc000000u) == 0x14000000u) {
// B (imm26:0..25)
delta = (uint32_t*)from_addr - ins_ptr;
if (((ins ^ (uint32_t)delta) & 0x01ffffffu) == 0) {
delta = (uint32_t*)to_addr - ins_ptr;
if (((delta + 0x02000000) >> 26) != 0) {
abort(); // branch target out of range
}
*ins_ptr = (ins & 0xfc000000u) | ((uint32_t)delta & 0x03ffffffu);
ret++;
if (!veneer) {
veneer = p;
}
}
} else if ((ins & 0xff000000u) == 0x54000000u ||
(ins & 0x7e000000u) == 0x34000000u) {
// B.cond, CBZ, CBNZ (imm19:5..23)
delta = (uint32_t*)from_addr - ins_ptr;
if (((ins ^ ((uint32_t)delta << 5)) & 0x00ffffe0u) == 0) {
delta = (uint32_t*)to_addr - ins_ptr;
if (((delta + 0x40000) >> 19) != 0) {
if (veneer) {
delta = (uint32_t*)veneer - ins_ptr;
if (((delta + 0x40000) >> 19) != 0) {
abort(); // branch target out of range
}
} else {
abort(); // branch target out of range
}
}
*ins_ptr = (ins & 0xff00001fu) | (((uint32_t)delta & 0x7ffffu) << 5);
ret++;
}
} else if ((ins & 0x7e000000u) == 0x36000000u) {
// TBZ, TBNZ (imm14:5..18)
delta = (uint32_t*)from_addr - ins_ptr;
if (((ins ^ ((uint32_t)delta << 5)) & 0x0007ffe0u) == 0) {
delta = (uint32_t*)to_addr - ins_ptr;
if (((delta + 0x2000) >> 14) != 0) {
if (veneer) {
delta = (uint32_t*)veneer - ins_ptr;
if (((delta + 0x2000) >> 14) != 0) {
abort(); // branch target out of range
}
} else {
abort(); // branch target out of range
}
}
*ins_ptr = (ins & 0xfff8001fu) | (((uint32_t)delta & 0x3fffu) << 5);
ret++;
}
}
}
JIT_CACHE_FLUSH(code, (char*)code + size);
#ifdef HAVE_VALGRIND
VALGRIND_DISCARD_TRANSLATIONS(code, size);
#endif
return ret;
}
static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
{
return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
}
static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr)
{
const void *link_addr;
size_t prologue_size;
/* Skip prologue. */
// TODO: don't hardcode this ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
prologue_size = 0;
#else
// sub sp, sp, #0x20
prologue_size = 4;
#endif
} else if (GCC_GLOBAL_REGS) {
// stp x29, x30, [sp, # -SPAD]!
prologue_size = 4;
} else {
// stp x29, x30, [sp, # -NR_SPAD]! // stack alignment
// stp FP, RX, T2
// mov FP, FCARG1x
prologue_size = 12;
}
link_addr = (const void*)((const char*)t->code_start + prologue_size);
if (timeout_exit_addr) {
/* Check timeout for links to LOOP */
| MEM_LOAD_8_ZTS ldrb, TMP1w, executor_globals, vm_interrupt, TMP1
| cbz TMP1w, &link_addr
| b &timeout_exit_addr
} else {
| b &link_addr
}
return 1;
}
static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline)
{
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
if (!original_handler) {
| JMP_IP TMP1
} else {
| ldr REG0, EX->func
| ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
| ldr REG0, [IP, REG0]
| br REG0
}
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
if (!original_handler) {
| JMP_IP TMP1
} else {
| ldr REG0, EX->func
| ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
| ldr REG0, [IP, REG0]
| br REG0
}
} else {
if (original_handler) {
| mov FCARG1x, FP
| ldr REG0, EX->func
| ldr REG0, [REG0, #offsetof(zend_op_array, reserved[zend_func_info_rid])]
| ldr REG0, [REG0, #offsetof(zend_jit_op_array_trace_extension, offset)]
| ldr REG0, [IP, REG0]
| blr REG0
}
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
if (!original_handler || !opline ||
(opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF
&& opline->opcode != ZEND_GENERATOR_RETURN
&& opline->opcode != ZEND_GENERATOR_CREATE
&& opline->opcode != ZEND_YIELD
&& opline->opcode != ZEND_YIELD_FROM)) {
| mov RETVALx, #2 // ZEND_VM_LEAVE
}
| ret
}
return 1;
}
static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, type, &exit_addr, ZREG_TMP1
return 1;
}
static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, var+offsetof(zval, u1.v.type), TMP1
| cmp TMP1w, #IS_STRING
| bhs &exit_addr
return 1;
}
static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!exit_addr) {
return 0;
}
| GET_ZVAL_LVAL ZREG_FCARG1, var_addr, TMP1
if (op_info & MAY_BE_ARRAY_PACKED) {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| beq &exit_addr
} else {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| bne &exit_addr
}
return 1;
}
static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
{
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
size_t offset = jit_extension->offset;
const void *handler =
(zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, FP
}
| EXT_CALL handler, REG0
if (may_throw
&& opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF) {
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbnz REG0, ->exception_handler
}
while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
trace++;
}
if (!GCC_GLOBAL_REGS
&& (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_GENERATOR_CREATE) {
| MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1
}
}
if (zend_jit_trace_may_exit(op_array, opline)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_GENERATOR_CREATE) {
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
/* this check should be handled by the following OPLINE guard or jmp [IP] */
| LOAD_ADDR TMP1, zend_jit_halt_op
| cmp IP, TMP1
| beq ->trace_halt
#endif
} else if (GCC_GLOBAL_REGS) {
| cbz IP, ->trace_halt
} else {
| tst RETVALw, RETVALw
| blt ->trace_halt
}
} else if (opline->opcode == ZEND_EXIT ||
opline->opcode == ZEND_GENERATOR_RETURN ||
opline->opcode == ZEND_YIELD ||
opline->opcode == ZEND_YIELD_FROM) {
| b ->trace_halt
}
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
const zend_op *next_opline = trace->opline;
const zend_op *exit_opline = NULL;
uint32_t exit_point;
const void *exit_addr;
uint32_t old_info = 0;
uint32_t old_res_info = 0;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
if (zend_is_smart_branch(opline)) {
bool exit_if_true = 0;
exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
} else {
switch (opline->opcode) {
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_JMP_NULL:
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
exit_opline = (trace->opline == opline + 1) ?
OP_JMP_ADDR(opline, opline->op2) :
opline + 1;
break;
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
exit_opline = (trace->opline == opline + 1) ?
ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
opline + 1;
break;
}
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
}
break;
}
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
}
exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
}
break;
}
if (!exit_addr) {
return 0;
}
| CMP_IP next_opline, TMP1, TMP2
| bne &exit_addr
}
}
zend_jit_set_last_valid_opline(trace->opline);
return 1;
}
static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw)
{
const void *handler;
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
handler = zend_get_opcode_handler_func(opline);
} else {
handler = opline->handler;
}
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, FP
}
| EXT_CALL handler, REG0
if (may_throw) {
zend_jit_check_exception(Dst);
}
/* Skip the following OP_DATA */
switch (opline->opcode) {
case ZEND_ASSIGN_DIM:
case ZEND_ASSIGN_OBJ:
case ZEND_ASSIGN_STATIC_PROP:
case ZEND_ASSIGN_DIM_OP:
case ZEND_ASSIGN_OBJ_OP:
case ZEND_ASSIGN_STATIC_PROP_OP:
case ZEND_ASSIGN_STATIC_PROP_REF:
case ZEND_ASSIGN_OBJ_REF:
zend_jit_set_last_valid_opline(opline + 2);
break;
default:
zend_jit_set_last_valid_opline(opline + 1);
break;
}
return 1;
}
static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
{
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_RETURN) {
/* Use inlined HYBRID VM handler */
const void *handler = opline->handler;
| ADD_HYBRID_SPAD
| EXT_JMP handler, REG0
} else {
const void *handler = zend_get_opcode_handler_func(opline);
| EXT_CALL handler, REG0
| ADD_HYBRID_SPAD
| JMP_IP TMP1
}
} else {
const void *handler = opline->handler;
if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
} else {
| mov FCARG1x, FP
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
}
| EXT_JMP handler, REG0
}
zend_jit_reset_last_valid_opline();
return 1;
}
static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline)
{
uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| CMP_IP opline, TMP1, TMP2
| bne &exit_addr
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label)
{
| b =>target_label
return 1;
}
static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label)
{
| CMP_IP next_opline, TMP1, TMP2
| bne =>target_label
zend_jit_set_last_valid_opline(next_opline);
return 1;
}
#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
| NIY // TODO
return 1;
}
#endif
static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
#ifdef CONTEXT_THREADED_JIT
return zend_jit_context_threaded_call(Dst, opline, next_block);
#else
return zend_jit_tail_handler(Dst, opline);
#endif
}
static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
{
ZEND_ASSERT(Z_MODE(src) == IS_REG);
ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| SET_ZVAL_LVAL_FROM_REG dst, Rx(Z_REG(src)), TMP1
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
| SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| SET_ZVAL_DVAL dst, Z_REG(src), ZREG_TMP1
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
ZEND_ASSERT(Z_MODE(dst) == IS_REG);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| GET_ZVAL_LVAL Z_REG(dst), src, TMP1
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| GET_ZVAL_DVAL Z_REG(dst), src, ZREG_TMP1
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type)
{
zend_jit_addr src = ZEND_ADDR_REG(reg);
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
return zend_jit_spill_store(Dst, src, dst, info, set_type);
}
static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
| SET_ZVAL_TYPE_INFO dst, type, TMP1w, TMP2
return 1;
}
static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
return zend_jit_spill_store(Dst, src, dst, info, 1);
}
return 1;
}
static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
bool set_type = 1;
if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
(old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
set_type = 0;
}
}
return zend_jit_spill_store(Dst, src, dst, info, set_type);
}
return 1;
}
static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg)
{
zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
zend_jit_addr dst = ZEND_ADDR_REG(reg);
return zend_jit_load_reg(Dst, src, dst, info);
}
static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, uint8_t op_type, zend_jit_addr addr, znode_op op)
{
if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
| SET_ZVAL_TYPE_INFO dst, IS_UNDEF, TMP1w, TMP2
}
return 1;
}
static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
if (!zend_jit_same_addr(src, dst)) {
if (Z_MODE(src) == IS_REG) {
if (Z_MODE(dst) == IS_REG) {
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| mov Rx(Z_REG(dst)), Rx(Z_REG(src))
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| fmov Rd(Z_REG(dst)-ZREG_V0), Rd(Z_REG(src)-ZREG_V0)
} else {
ZEND_UNREACHABLE();
}
if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(Dst, dst, var_addr, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else if (Z_MODE(dst) == IS_MEM_ZVAL) {
if (!Z_LOAD(src) && !Z_STORE(src)) {
if (!zend_jit_spill_store(Dst, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(src) == IS_MEM_ZVAL) {
if (Z_MODE(dst) == IS_REG) {
if (!zend_jit_load_reg(Dst, src, dst, info)) {
return 0;
}
} else {
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(Dst, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
return 1;
}
static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline)
{
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1
if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
if (!zend_jit_save_call_chain(Dst, -1)) {
return 0;
}
}
ZEND_ASSERT(opline);
if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
&& (opline-1)->opcode != ZEND_FETCH_LIST_R
&& ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
| IF_NOT_ZVAL_REFCOUNTED val_addr, >2, ZREG_TMP1, ZREG_TMP2
| GET_ZVAL_PTR TMP1, val_addr, TMP2
| GC_ADDREF TMP1, TMP2w
|2:
}
| LOAD_IP_ADDR (opline - 1)
| b ->trace_escape
|1:
return 1;
}
static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
if (reg == ZREG_LONG_MIN_MINUS_1) {
uint64_t val = 0xc3e0000000000000;
| SET_ZVAL_LVAL dst, val, TMP1, TMP2
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2
} else if (reg == ZREG_LONG_MIN) {
uint64_t val = 0x8000000000000000;
| SET_ZVAL_LVAL dst, val, TMP1, TMP2
| SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2
} else if (reg == ZREG_LONG_MAX) {
uint64_t val = 0x7fffffffffffffff;
| SET_ZVAL_LVAL dst, val, TMP1, TMP2
| SET_ZVAL_TYPE_INFO dst, IS_LONG, TMP1w, TMP2
} else if (reg == ZREG_LONG_MAX_PLUS_1) {
uint64_t val = 0x43e0000000000000;
| SET_ZVAL_LVAL dst, val, TMP1, TMP2
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE, TMP1w, TMP2
} else if (reg == ZREG_NULL) {
| SET_ZVAL_TYPE_INFO dst, IS_NULL, TMP1w, TMP2
} else if (reg == ZREG_ZVAL_TRY_ADDREF) {
| IF_NOT_ZVAL_REFCOUNTED dst, >1, ZREG_TMP1, ZREG_TMP2
| GET_ZVAL_PTR TMP1, dst, TMP2
| GC_ADDREF TMP1, TMP2w
|1:
} else if (reg == ZREG_ZVAL_COPY_GPR0) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF -1, REG1w, REG2, TMP1w
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_free_trampoline(dasm_State **Dst)
{
| // if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
| ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)]
| TST_32_WITH_CONST TMP1w, ZEND_ACC_CALL_VIA_TRAMPOLINE, TMP2w
| beq >1
| mov FCARG1x, REG0
| EXT_CALL zend_jit_free_trampoline_helper, REG0
|1:
return 1;
}
static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) {
return 0;
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
| LONG_ADD_SUB_WITH_IMM adds, op1_def_addr, Z_L(1), TMP1, TMP2
} else {
| LONG_ADD_SUB_WITH_IMM subs, op1_def_addr, Z_L(1), TMP1, TMP2
}
if (may_overflow &&
(((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) ||
((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) {
int32_t exit_point;
const void *exit_addr;
zend_jit_trace_stack *stack;
uint32_t old_op1_info, old_res_info = 0;
stack = JIT_G(current_frame)->stack;
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1);
} else {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1);
}
if (opline->result_type != IS_UNUSED) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
if (opline->opcode == ZEND_PRE_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1);
} else if (opline->opcode == ZEND_PRE_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1);
} else if (opline->opcode == ZEND_POST_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX);
} else if (opline->opcode == ZEND_POST_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN);
}
}
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| bvs &exit_addr
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
if (opline->result_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
} else if (may_overflow) {
| bvs >1
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
|.cold_code
|1:
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
uint64_t val = 0x43e0000000000000;
if (Z_MODE(op1_def_addr) == IS_REG) {
| LOAD_64BIT_VAL TMP1, val
| fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1
} else {
| SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1
}
} else {
uint64_t val = 0xc3e0000000000000;
if (Z_MODE(op1_def_addr) == IS_REG) {
| LOAD_64BIT_VAL TMP1, val
| fmov Rd(Z_REG(op1_def_addr)-ZREG_V0), TMP1
} else {
| SET_ZVAL_LVAL op1_def_addr, val, TMP2, TMP1
}
}
if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE, TMP1w, TMP2
}
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
| b >3
|.code
} else {
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
|.cold_code
|2:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| SET_EX_OPLINE opline, REG0
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2, ZREG_TMP1
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2
op1_info |= MAY_BE_NULL;
}
|2:
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| // ZVAL_DEREF(var_ptr);
if (op1_info & MAY_BE_REF) {
| IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >2, TMP1w
| GET_Z_PTR FCARG1x, FCARG1x
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
| cbz TMP1, >1
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR FCARG2x, res_addr
} else {
| mov FCARG2x, xzr
}
if (opline->opcode == ZEND_PRE_INC) {
| EXT_CALL zend_jit_pre_inc_typed_ref, REG0
} else if (opline->opcode == ZEND_PRE_DEC) {
| EXT_CALL zend_jit_pre_dec_typed_ref, REG0
} else if (opline->opcode == ZEND_POST_INC) {
| EXT_CALL zend_jit_post_inc_typed_ref, REG0
} else if (opline->opcode == ZEND_POST_DEC) {
| EXT_CALL zend_jit_post_dec_typed_ref, REG0
} else {
ZEND_UNREACHABLE();
}
zend_jit_check_exception(Dst);
| b >3
|1:
| add FCARG1x, FCARG1x, #offsetof(zend_reference, val)
|2:
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
| ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF op1_info, REG0w, REG2, TMP1w
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2x, res_addr
| EXT_CALL zend_jit_pre_inc, REG0
} else {
| EXT_CALL increment_function, REG0
}
} else {
if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2x, res_addr
| EXT_CALL zend_jit_pre_dec, REG0
} else {
| EXT_CALL decrement_function, REG0
}
}
if (may_throw) {
zend_jit_check_exception(Dst);
}
} else {
zend_reg tmp_reg;
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
if (Z_MODE(op1_def_addr) == IS_REG) {
tmp_reg = Z_REG(op1_def_addr);
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
tmp_reg = Z_REG(op1_addr);
} else {
tmp_reg = ZREG_FPR0;
}
| GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
uint64_t val = 0x3ff0000000000000; // 1.0
| LOAD_64BIT_VAL TMP1, val
| fmov FPTMP, TMP1
| fadd Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP
} else {
uint64_t val = 0x3ff0000000000000; // 1.0
| LOAD_64BIT_VAL TMP1, val
| fmov FPTMP, TMP1
| fsub Rd(tmp_reg-ZREG_V0), Rd(tmp_reg-ZREG_V0), FPTMP
}
| SET_ZVAL_DVAL op1_def_addr, tmp_reg, ZREG_TMP1
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF op1_def_info, REG0w, REG1, TMP1w
}
}
| b >3
|.code
}
|3:
if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg)
{
if ((opline+1)->opcode == ZEND_OP_DATA
&& ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV))
&& JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) {
return 1;
}
return
((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) ||
((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg);
}
static int zend_jit_math_long_long(dasm_State **Dst,
const zend_op *opline,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow)
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
zend_reg tmp_reg = ZREG_REG0;
bool use_ovf_flag = 1;
if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) {
if (may_overflow && (res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) {
result_reg = ZREG_REG0;
} else {
result_reg = Z_REG(res_addr);
}
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) {
result_reg = Z_REG(op1_addr);
} else if (Z_REG(res_addr) != ZREG_REG0) {
result_reg = ZREG_REG0;
} else {
/* ASSIGN_DIM_OP */
result_reg = ZREG_FCARG1;
tmp_reg = ZREG_FCARG1;
}
if (opcode == ZEND_MUL &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
Z_LVAL_P(Z_ZV(op2_addr)) == 2) {
if (Z_MODE(op1_addr) == IS_REG) {
| adds Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr))
} else {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| adds Rx(result_reg), Rx(result_reg), Rx(result_reg)
}
} else if (opcode == ZEND_MUL &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
!may_overflow &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
| lsl Rx(result_reg), Rx(result_reg), TMP1
} else if (opcode == ZEND_MUL &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
Z_LVAL_P(Z_ZV(op1_addr)) == 2) {
if (Z_MODE(op2_addr) == IS_REG) {
| adds Rx(result_reg), Rx(Z_REG(op2_addr)), Rx(Z_REG(op2_addr))
} else {
| GET_ZVAL_LVAL result_reg, op2_addr, TMP1
| adds Rx(result_reg), Rx(result_reg), Rx(result_reg)
}
} else if (opcode == ZEND_MUL &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
!may_overflow &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) {
| GET_ZVAL_LVAL result_reg, op2_addr, TMP1
| mov TMP1, #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr)))
| lsl Rx(result_reg), Rx(result_reg), TMP1
} else if (opcode == ZEND_DIV &&
(Z_MODE(op2_addr) == IS_CONST_ZVAL &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| asr Rx(result_reg), Rx(result_reg), #zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
#if 0
/* x86 specific optimizations through LEA instraction are not supported on ARM */
} else if (opcode == ZEND_ADD &&
!may_overflow &&
Z_MODE(op1_addr) == IS_REG &&
Z_MODE(op2_addr) == IS_CONST_ZVAL) {
| NIY // TODO: test
} else if (opcode == ZEND_ADD &&
!may_overflow &&
Z_MODE(op2_addr) == IS_REG &&
Z_MODE(op1_addr) == IS_CONST_ZVAL) {
| NIY // TODO: test
} else if (opcode == ZEND_SUB &&
!may_overflow &&
Z_MODE(op1_addr) == IS_REG &&
Z_MODE(op2_addr) == IS_CONST_ZVAL) {
| NIY // TODO: test
#endif
} else if (opcode == ZEND_MUL) {
| GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1
| GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
| mul Rx(result_reg), TMP1, TMP2
if(may_overflow) {
/* Use 'smulh' to get the upper 64 bits fo the 128-bit result.
* For signed multiplication, the top 65 bits of the result will contain
* either all zeros or all ones if no overflow occurred.
* Flag: bne -> overflow. beq -> no overflow.
*/
use_ovf_flag = 0;
| smulh TMP1, TMP1, TMP2
| cmp TMP1, Rx(result_reg), asr #63
}
} else {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
may_overflow = 0;
} else if (same_ops && opcode != ZEND_DIV) {
| LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg)
} else {
| LONG_MATH opcode, result_reg, op2_addr, TMP1
}
}
if (may_overflow) {
if (res_info & MAY_BE_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
if (use_ovf_flag) {
| bvs &exit_addr
} else {
| bne &exit_addr
}
if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) {
| mov Rx(Z_REG(res_addr)), Rx(result_reg)
}
} else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
if (use_ovf_flag) {
| bvc &exit_addr
} else {
| beq &exit_addr
}
} else {
ZEND_UNREACHABLE();
}
} else {
if (res_info & MAY_BE_LONG) {
if (use_ovf_flag) {
| bvs >1
} else {
| bne >1
}
} else {
if (use_ovf_flag) {
| bvc >1
} else {
| beq >1
}
}
}
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) {
| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
}
}
}
if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) {
zend_reg tmp_reg1 = ZREG_FPR0;
zend_reg tmp_reg2 = ZREG_FPR1;
if (res_info & MAY_BE_LONG) {
|.cold_code
|1:
}
do {
if ((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) ||
(Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1)) {
if (opcode == ZEND_ADD) {
uint64_t val = 0x43e0000000000000;
if (Z_MODE(res_addr) == IS_REG) {
| LOAD_64BIT_VAL TMP1, val
| fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1
} else {
| SET_ZVAL_LVAL res_addr, val, TMP2, TMP1
}
break;
} else if (opcode == ZEND_SUB) {
uint64_t val = 0xc3e0000000000000;
if (Z_MODE(res_addr) == IS_REG) {
| LOAD_64BIT_VAL TMP1, val
| fmov Rd(Z_REG(res_addr)-ZREG_V0), TMP1
} else {
| SET_ZVAL_LVAL res_addr, val, TMP2, TMP1
}
break;
}
}
| DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg, ZREG_TMP1
| DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg, ZREG_TMP1
| DOUBLE_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2
| SET_ZVAL_DVAL res_addr, tmp_reg1, ZREG_TMP1
} while (0);
if (Z_MODE(res_addr) == IS_MEM_ZVAL
&& (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
}
if (res_info & MAY_BE_LONG) {
| b >2
|.code
}
|2:
}
return 1;
}
static int zend_jit_math_long_double(dasm_State **Dst,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
zend_reg result_reg =
(Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_FPR0;
zend_reg op2_reg;
| DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, ZREG_TMP1, ZREG_TMP2
if (Z_MODE(op2_addr) == IS_REG) {
op2_reg = Z_REG(op2_addr);
} else {
op2_reg = ZREG_FPTMP;
| GET_ZVAL_DVAL op2_reg, op2_addr, ZREG_TMP1
}
| DOUBLE_MATH_REG opcode, result_reg, result_reg, op2_reg
| SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
}
}
return 1;
}
static int zend_jit_math_double_long(dasm_State **Dst,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
zend_reg result_reg, op1_reg, op2_reg;
if (zend_is_commutative(opcode)
&& (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) {
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
} else {
result_reg = ZREG_FPR0;
}
| DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, ZREG_TMP1, ZREG_TMP2
if (Z_MODE(op1_addr) == IS_REG) {
op1_reg = Z_REG(op1_addr);
} else {
op1_reg = ZREG_FPTMP;
| GET_ZVAL_DVAL op1_reg, op1_addr, ZREG_TMP1
}
| DOUBLE_MATH_REG opcode, result_reg, result_reg, op1_reg
} else {
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
} else {
result_reg = ZREG_FPR0;
}
if (Z_MODE(op1_addr) == IS_REG) {
op1_reg = Z_REG(op1_addr);
} else {
| GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1
op1_reg = result_reg;
}
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
} else {
op2_reg = ZREG_FPTMP;
| DOUBLE_GET_ZVAL_LVAL op2_reg, op2_addr, ZREG_TMP1, ZREG_TMP2
| DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg
}
}
| SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
}
}
}
return 1;
}
static int zend_jit_math_double_double(dasm_State **Dst,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg, op1_reg, op2_reg;
zend_jit_addr val_addr;
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
} else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) {
result_reg = Z_REG(op2_addr);
} else {
result_reg = ZREG_FPR0;
}
if (Z_MODE(op1_addr) == IS_REG) {
op1_reg = Z_REG(op1_addr);
val_addr = op2_addr;
} else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) {
op1_reg = Z_REG(op2_addr);
val_addr = op1_addr;
} else {
| GET_ZVAL_DVAL result_reg, op1_addr, ZREG_TMP1
op1_reg = result_reg;
val_addr = op2_addr;
}
if ((opcode == ZEND_MUL) &&
Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
| DOUBLE_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg
} else {
if (same_ops) {
op2_reg = op1_reg;
} else if (Z_MODE(val_addr) == IS_REG) {
op2_reg = Z_REG(val_addr);
} else {
op2_reg = ZREG_FPTMP;
| GET_ZVAL_DVAL op2_reg, val_addr, ZREG_TMP1
}
| DOUBLE_MATH_REG opcode, result_reg, op1_reg, op2_reg
}
| SET_ZVAL_DVAL res_addr, result_reg, ZREG_TMP1
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
}
}
}
return 1;
}
static int zend_jit_math_helper(dasm_State **Dst,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow,
int may_throw)
/* Labels: 1,2,3,4,5,6 */
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
}
}
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
if (op2_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1, ZREG_TMP1
|.cold_code
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1
}
if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| b >5
|.code
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
}
}
if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
|3:
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops) {
| IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6, ZREG_TMP1
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| b >5
}
if (!same_ops) {
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
}
if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| b >5
}
|.code
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops && (op2_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & MAY_BE_DOUBLE) {
|.cold_code
}
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
}
if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
if (op2_info & MAY_BE_DOUBLE) {
| b >5
|.code
}
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6, ZREG_TMP1
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops && (op1_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6, ZREG_TMP1
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
}
|1:
if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
}
if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
| b >5
|.code
}
}
}
|5:
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
|.cold_code
}
|6:
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR CARG3, op2_addr
| SET_EX_OPLINE opline, REG0
if (opcode == ZEND_ADD) {
| EXT_CALL add_function, REG0
} else if (opcode == ZEND_SUB) {
| EXT_CALL sub_function, REG0
} else if (opcode == ZEND_MUL) {
| EXT_CALL mul_function, REG0
} else if (opcode == ZEND_DIV) {
| EXT_CALL div_function, REG0
} else {
ZEND_UNREACHABLE();
}
| FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
| FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
| MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
| cbnz TMP2, ->exception_handler_free_op2
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
return 0;
}
}
if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
| b <5
|.code
}
}
return 1;
}
static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)));
if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
{
if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) {
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
} else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
} else {
| GET_ZVAL_LVAL ZREG_REG0, op2_addr, TMP1
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
| mov FCARG2x, REG0
}
| EXT_CALL zend_jit_add_arrays_helper, REG0
| SET_ZVAL_PTR res_addr, RETVALx, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX, TMP1w, TMP2
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
return 1;
}
static int zend_jit_long_math_helper(dasm_State **Dst,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
zend_ssa_range *op1_range,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_throw)
/* Labels: 6 */
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6, ZREG_TMP1
}
if (Z_MODE(res_addr) == IS_REG) {
if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR)
&& opline->op2_type != IS_CONST) {
result_reg = ZREG_REG0;
} else {
result_reg = Z_REG(res_addr);
}
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
} else if (Z_REG(res_addr) != ZREG_REG0) {
result_reg = ZREG_REG0;
} else {
/* ASSIGN_DIM_OP */
result_reg = ZREG_FCARG1;
}
if (opcode == ZEND_SL) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
| mov Rx(result_reg), xzr
} else {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, REG0
| b ->negative_shift
}
} else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) {
| add Rx(result_reg), Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr))
} else {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| lsl Rx(result_reg), Rx(result_reg), #op2_lval
}
} else {
zend_reg op2_reg;
if (Z_MODE(op2_addr) == IS_REG) {
op2_reg = Z_REG(op2_addr);
} else {
op2_reg = ZREG_TMP2;
| GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
}
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
| cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8)
| bhs >1
|.cold_code
|1:
| mov Rx(result_reg), xzr
| cmp Rx(op2_reg), xzr
| bgt >1
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, REG0
| b ->negative_shift
|.code
}
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| lsl Rx(result_reg), Rx(result_reg), Rx(op2_reg)
|1:
}
} else if (opcode == ZEND_SR) {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
| asr Rx(result_reg), Rx(result_reg), #((SIZEOF_ZEND_LONG * 8) - 1)
} else {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, REG0
| b ->negative_shift
}
} else {
| asr Rx(result_reg), Rx(result_reg), #op2_lval
}
} else {
zend_reg op2_reg;
if (Z_MODE(op2_addr) == IS_REG) {
op2_reg = Z_REG(op2_addr);
} else {
op2_reg = ZREG_TMP2;
| GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
}
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
| cmp Rx(op2_reg), #(SIZEOF_ZEND_LONG*8)
| bhs >1
|.cold_code
|1:
| cmp Rx(op2_reg), xzr
| mov Rx(op2_reg), #((SIZEOF_ZEND_LONG * 8) - 1)
| bgt >1
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, REG0
| b ->negative_shift
|.code
}
|1:
| asr Rx(result_reg), Rx(result_reg), Rx(op2_reg)
}
} else if (opcode == ZEND_MOD) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (op2_lval == 0) {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, REG0
| b ->mod_by_zero
} else if (op2_lval == -1) {
| mov Rx(result_reg), xzr
} else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
zval tmp;
zend_jit_addr tmp_addr;
/* Optimisation for mod of power of 2 */
ZVAL_LONG(&tmp, op2_lval - 1);
tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp);
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, TMP1
} else {
| GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1
| GET_ZVAL_LVAL ZREG_TMP2, op2_addr, TMP2
| sdiv Rx(result_reg), TMP1, TMP2
| msub Rx(result_reg), Rx(result_reg), TMP2, TMP1
}
} else {
zend_reg op2_reg;
if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, TMP2, Rx(Z_REG(op2_addr)), Z_OFFSET(op2_addr), TMP2
op2_reg = ZREG_TMP2;
} else {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_REG);
op2_reg = Z_REG(op2_addr);
}
if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
| cbz Rx(op2_reg), >1
|.cold_code
|1:
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, REG0
| b ->mod_by_zero
|.code
}
/* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
| cmn Rx(op2_reg), #1
| beq >1
|.cold_code
|1:
| SET_ZVAL_LVAL_FROM_REG res_addr, xzr, TMP1
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
}
}
}
| b >5
|.code
}
| GET_ZVAL_LVAL ZREG_TMP1, op1_addr, TMP1
| sdiv Rx(result_reg), TMP1, Rx(op2_reg)
| msub Rx(result_reg), Rx(result_reg), Rx(op2_reg), TMP1
}
} else if (same_ops) {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| LONG_MATH_REG opcode, Rx(result_reg), Rx(result_reg), Rx(result_reg)
} else {
| GET_ZVAL_LVAL result_reg, op1_addr, TMP1
| LONG_MATH opcode, result_reg, op2_addr, TMP1
}
if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) {
| SET_ZVAL_LVAL_FROM_REG res_addr, Rx(result_reg), TMP1
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
}
}
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if ((op1_info & MAY_BE_LONG) &&
(op2_info & MAY_BE_LONG)) {
|.cold_code
}
|6:
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1x, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR CARG3, op2_addr
| SET_EX_OPLINE opline, REG0
if (opcode == ZEND_BW_OR) {
| EXT_CALL bitwise_or_function, REG0
} else if (opcode == ZEND_BW_AND) {
| EXT_CALL bitwise_and_function, REG0
} else if (opcode == ZEND_BW_XOR) {
| EXT_CALL bitwise_xor_function, REG0
} else if (opcode == ZEND_SL) {
| EXT_CALL shift_left_function, REG0
} else if (opcode == ZEND_SR) {
| EXT_CALL shift_right_function, REG0
} else if (opcode == ZEND_MOD) {
| EXT_CALL mod_function, REG0
} else {
ZEND_UNREACHABLE();
}
if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
/* compound assignment may decrement "op2" refcount */
op2_info |= MAY_BE_RC1;
}
| FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
| FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
| MEM_LOAD_64_ZTS ldr, TMP2, executor_globals, exception, TMP1
| cbnz TMP2, ->exception_handler_free_op2
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
return 0;
}
}
if ((op1_info & MAY_BE_LONG) &&
(op2_info & MAY_BE_LONG)) {
| b >5
|.code
}
}
|5:
return 1;
}
static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
if (!zend_jit_long_math_helper(Dst, opline, opline->opcode,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_concat_helper(dasm_State **Dst,
const zend_op *opline,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
int may_throw)
{
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1
}
if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6, ZREG_TMP1
}
if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) {
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
| EXT_CALL zend_jit_fast_assign_concat_helper, REG0
/* concatenation with itself may reduce refcount */
op2_info |= MAY_BE_RC1;
} else {
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
| LOAD_ZVAL_ADDR CARG3, op2_addr
if (op1_type == IS_CV || op1_type == IS_CONST) {
| EXT_CALL zend_jit_fast_concat_helper, REG0
} else {
| EXT_CALL zend_jit_fast_concat_tmp_helper, REG0
}
}
/* concatenation with empty string may increase refcount */
op2_info |= MAY_BE_RCN;
| FREE_OP op2_type, op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
|5:
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
(op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
|.cold_code
|6:
}
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
} else {
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
}
| LOAD_ZVAL_ADDR CARG3, op2_addr
| SET_EX_OPLINE opline, REG0
| EXT_CALL concat_function, REG0
/* concatenation with empty string may increase refcount */
op1_info |= MAY_BE_RCN;
op2_info |= MAY_BE_RCN;
| FREE_OP op1_type, op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
| FREE_OP op2_type, op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
| b <5
|.code
}
}
return 1;
}
static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
}
static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr)
/* Labels: 1,2,3,4,5 */
{
zend_jit_addr op2_addr = OP2_ADDR();
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& type == BP_VAR_R
&& !exit_addr) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (op2_info & MAY_BE_LONG) {
bool op2_loaded = 0;
bool packed_loaded = 0;
bool bad_packed_key = 0;
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
| // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1
}
if (op1_info & MAY_BE_PACKED_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| beq &exit_addr
} else {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| bne &exit_addr
}
}
if (type == BP_VAR_W) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
op2_loaded = 1;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
zend_long val = -1;
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
val = Z_LVAL_P(Z_ZV(op2_addr));
if (val >= 0 && val < HT_MAX_SIZE) {
packed_loaded = 1;
} else {
bad_packed_key = 1;
}
} else {
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
op2_loaded = 1;
}
packed_loaded = 1;
}
if (dim_type == IS_UNDEF && type == BP_VAR_W) {
/* don't generate "fast" code for packed array */
packed_loaded = 0;
}
if (packed_loaded) {
| // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| beq >4 // HASH_FIND
}
| // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
| ldr REG0w, [FCARG1x, #offsetof(zend_array, nNumUsed)]
if (val == 0) {
| cmp REG0, xzr
} else if (val > 0 && !op2_loaded) {
| CMP_64_WITH_CONST REG0, val, TMP1
} else {
| cmp REG0, FCARG2x
}
if (type == BP_JIT_IS) {
if (not_found_exit_addr) {
| bls &not_found_exit_addr
} else {
| bls >9 // NOT_FOUND
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| bls &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| bls &not_found_exit_addr
} else if (type == BP_VAR_RW && not_found_exit_addr) {
| bls &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| bls >7 // NOT_FOUND
} else {
| bls >2 // NOT_FOUND
}
| // _ret = &_ht->arPacked[_h].val;
if (val >= 0) {
| ldr REG0, [FCARG1x, #offsetof(zend_array, arPacked)]
if (val != 0) {
| ADD_SUB_64_WITH_CONST add, REG0, REG0, (val * sizeof(zval)), TMP1
}
} else {
| ldr TMP1, [FCARG1x, #offsetof(zend_array, arPacked)]
| add REG0, TMP1, FCARG2x, lsl #4
}
}
}
switch (type) {
case BP_JIT_IS:
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if (packed_loaded) {
| b >5
}
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
}
if (packed_loaded) {
| EXT_CALL _zend_hash_index_find, REG0
} else {
| EXT_CALL zend_hash_index_find, REG0
}
| mov REG0, RETVALx
if (not_found_exit_addr) {
| cbz REG0, &not_found_exit_addr
} else {
| cbz REG0, >9 // NOT_FOUND
}
if (op2_info & MAY_BE_STRING) {
| b >5
}
} else if (packed_loaded) {
if (op2_info & MAY_BE_STRING) {
| b >5
}
} else if (not_found_exit_addr) {
| b &not_found_exit_addr
} else {
| b >9 // NOT_FOUND
}
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (packed_loaded) {
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
| IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
| IF_Z_TYPE REG0, IS_UNDEF, &exit_addr, TMP1w
}
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| IF_Z_TYPE REG0, IS_UNDEF, &not_found_exit_addr, TMP1w
} else if (type == BP_VAR_IS && found_exit_addr) {
| IF_Z_TYPE REG0, IS_UNDEF, >7, TMP1w // NOT_FOUND
} else {
| IF_Z_TYPE REG0, IS_UNDEF, >2, TMP1w // NOT_FOUND
}
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| b &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| b &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| b >7 // NOT_FOUND
} else {
| b >2 // NOT_FOUND
}
}
if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
}
if (packed_loaded) {
| EXT_CALL _zend_hash_index_find, REG0
} else {
| EXT_CALL zend_hash_index_find, REG0
}
| mov REG0, RETVALx
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| cbz REG0, &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| cbz REG0, &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| cbz REG0, >7 // NOT_FOUND
} else {
| cbz REG0, >2 // NOT_FOUND
}
}
|.cold_code
|2:
switch (type) {
case BP_VAR_R:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
| // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
| // retval = &EG(uninitialized_zval);
| UNDEFINED_OFFSET opline
| b >9
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
if (!not_found_exit_addr && !found_exit_addr) {
| // retval = &EG(uninitialized_zval);
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
| b >9
}
break;
default:
ZEND_UNREACHABLE();
}
|.code
break;
case BP_VAR_RW:
if (packed_loaded && !not_found_exit_addr) {
| IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w
}
if (!packed_loaded ||
!not_found_exit_addr ||
(op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
if (packed_loaded && not_found_exit_addr) {
|.cold_code
}
|2:
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
}
if (packed_loaded) {
| EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, REG0
} else {
| EXT_CALL zend_jit_hash_index_lookup_rw, REG0
}
| mov REG0, RETVALx
if (not_found_exit_addr) {
if (packed_loaded) {
| cbnz REG0, >8
| b &not_found_exit_addr
|.code
} else {
| cbz REG0, &not_found_exit_addr
}
} else {
| cbz REG0, >9
}
}
break;
case BP_VAR_W:
if (packed_loaded) {
| IF_NOT_Z_TYPE REG0, IS_UNDEF, >8, TMP1w
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
|2:
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
}
| EXT_CALL zend_hash_index_lookup, REG0
| mov REG0, RETVALx
}
break;
default:
ZEND_UNREACHABLE();
}
if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) {
| b >8
}
}
if (op2_info & MAY_BE_STRING) {
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
| // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
| IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3, ZREG_TMP1
}
| // offset_key = Z_STR_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
| // retval = zend_hash_find(ht, offset_key);
switch (type) {
case BP_JIT_IS:
if (opline->op2_type != IS_CONST) {
| ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)]
| cmp TMP1w, #((uint8_t) ('9'))
| ble >1
|.cold_code
|1:
| EXT_CALL zend_jit_symtable_find, REG0
| b >1
|.code
| EXT_CALL zend_hash_find, REG0
|1:
} else {
| EXT_CALL zend_hash_find_known_hash, REG0
}
| mov REG0, RETVALx
if (not_found_exit_addr) {
| cbz REG0, &not_found_exit_addr
} else {
| cbz REG0, >9 // NOT_FOUND
}
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (opline->op2_type != IS_CONST) {
| ldrb TMP1w, [FCARG2x, #offsetof(zend_string, val)]
| cmp TMP1w, #((uint8_t) ('9'))
| ble >1
|.cold_code
|1:
| EXT_CALL zend_jit_symtable_find, REG0
| b >1
|.code
| EXT_CALL zend_hash_find, REG0
|1:
} else {
| EXT_CALL zend_hash_find_known_hash, REG0
}
| mov REG0, RETVALx
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| cbz REG0, &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| cbz REG0, &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| cbz REG0, >7
} else {
| cbz REG0, >2 // NOT_FOUND
|.cold_code
|2:
switch (type) {
case BP_VAR_R:
// zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
| UNDEFINED_INDEX opline
| b >9
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
| // retval = &EG(uninitialized_zval);
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
| b >9
break;
default:
ZEND_UNREACHABLE();
}
|.code
}
break;
case BP_VAR_RW:
if (opline->op2_type != IS_CONST) {
| EXT_CALL zend_jit_symtable_lookup_rw, REG0
} else {
| EXT_CALL zend_jit_hash_lookup_rw, REG0
}
| mov REG0, RETVALx
if (not_found_exit_addr) {
| cbz REG0, &not_found_exit_addr
} else {
| cbz REG0, >9
}
break;
case BP_VAR_W:
if (opline->op2_type != IS_CONST) {
| EXT_CALL zend_jit_symtable_lookup_w, REG0
} else {
| EXT_CALL zend_hash_lookup, REG0
}
| mov REG0, RETVALx
break;
default:
ZEND_UNREACHABLE();
}
}
if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) {
|5:
if (op1_info & MAY_BE_ARRAY_OF_REF) {
| ZVAL_DEREF REG0, MAY_BE_REF, TMP1w
}
| ldrb TMP1w, [REG0,#offsetof(zval, u1.v.type)]
| cmp TMP1w, #IS_NULL
if (not_found_exit_addr) {
| ble &not_found_exit_addr
} else if (found_exit_addr) {
| bgt &found_exit_addr
} else {
| ble >9 // NOT FOUND
}
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
|.cold_code
|3:
}
if (type != BP_VAR_RW) {
| SET_EX_OPLINE opline, REG0
}
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
switch (type) {
case BP_VAR_R:
| LOAD_ZVAL_ADDR CARG3, res_addr
| EXT_CALL zend_jit_fetch_dim_r_helper, REG0
| mov REG0, RETVALx
| b >9
break;
case BP_JIT_IS:
| EXT_CALL zend_jit_fetch_dim_isset_helper, REG0
| mov REG0, RETVALx
if (not_found_exit_addr) {
| cbz REG0, &not_found_exit_addr
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
| b >8
}
} else if (found_exit_addr) {
| cbnz REG0, &found_exit_addr
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
| b >9
}
} else {
| cbnz REG0, >8
| b >9
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
| LOAD_ZVAL_ADDR CARG3, res_addr
| EXT_CALL zend_jit_fetch_dim_is_helper, REG0
| mov REG0, RETVALx
| b >9
break;
case BP_VAR_RW:
| EXT_CALL zend_jit_fetch_dim_rw_helper, REG0
| mov REG0, RETVALx
| cbnz REG0, >8
| b >9
break;
case BP_VAR_W:
| EXT_CALL zend_jit_fetch_dim_w_helper, REG0
| mov REG0, RETVALx
| cbnz REG0, >8
| b >9
break;
default:
ZEND_UNREACHABLE();
}
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
|.code
}
}
return 1;
}
static int zend_jit_simple_assign(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
int in_cold,
int save_r1,
bool check_exception)
/* Labels: 1,2,3 */
{
zend_reg tmp_reg;
if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_REG0) {
tmp_reg = ZREG_REG0;
} else {
/* ASSIGN_DIM */
tmp_reg = ZREG_FCARG1;
}
if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
zval *zv = Z_ZV(val_addr);
if (!res_addr) {
| ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0
} else {
| ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg, ZREG_TMP1, ZREG_FPR0
}
if (Z_REFCOUNTED_P(zv)) {
if (!res_addr) {
| ADDREF_CONST zv, TMP1, TMP2
} else {
| ADDREF_CONST_2 zv, TMP1, TMP2
}
}
} else {
if (val_info & MAY_BE_UNDEF) {
if (in_cold) {
| IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2, ZREG_TMP1
} else {
| IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1
|.cold_code
|1:
}
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
if (save_r1) {
| str FCARG1x, T1 // save
}
| SET_ZVAL_TYPE_INFO var_addr, IS_NULL, TMP1w, TMP2
if (res_addr) {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
}
if (opline) {
| SET_EX_OPLINE opline, Rx(tmp_reg)
}
ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
| LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr)
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (check_exception) {
| cbz RETVALx, ->exception_handler_undef
}
if (save_r1) {
| ldr FCARG1x, T1 // restore
}
| b >3
if (in_cold) {
|2:
} else {
|.code
}
}
if (val_info & MAY_BE_REF) {
if (val_type == IS_CV) {
ZEND_ASSERT(Z_REG(var_addr) != ZREG_REG2);
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_REG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR REG2, val_addr
}
| ZVAL_DEREF REG2, val_info, TMP1w
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0);
} else {
zend_jit_addr ref_addr;
if (in_cold) {
| IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1
} else {
| IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1, ZREG_TMP1
|.cold_code
|1:
}
if (Z_REG(val_addr) == ZREG_REG2) {
| str REG2, T1 // save
}
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR REG2, val_addr, TMP1
| GC_DELREF REG2, TMP1w
| // ZVAL_COPY_VALUE(return_value, &ref->val);
ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, offsetof(zend_reference, val));
if (!res_addr) {
| ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
} else {
| ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
| beq >2 // GC_DELREF() reached zero
| IF_NOT_REFCOUNTED REG2w, >3, TMP1w
if (!res_addr) {
| GC_ADDREF Rx(tmp_reg), TMP1w
} else {
| GC_ADDREF_2 Rx(tmp_reg), TMP1w
}
| b >3
|2:
if (res_addr) {
| IF_NOT_REFCOUNTED REG2w, >2, TMP1w
| GC_ADDREF Rx(tmp_reg), TMP1w
|2:
}
if (Z_REG(val_addr) == ZREG_REG2) {
| ldr REG2, T1 // restore
}
if (save_r1) {
| str FCARG1x, T1 // save
}
| GET_ZVAL_PTR FCARG1x, val_addr, TMP1
| EFREE_REFERENCE
if (save_r1) {
| ldr FCARG1x, T1 // restore
}
| b >3
if (in_cold) {
|1:
} else {
|.code
}
}
}
if (!res_addr) {
| ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
} else {
| ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_REG2, tmp_reg, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
if (val_type == IS_CV) {
if (!res_addr) {
| TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w
} else {
| TRY_ADDREF_2 val_info, REG2w, Rx(tmp_reg), TMP1w
}
} else {
if (res_addr) {
| TRY_ADDREF val_info, REG2w, Rx(tmp_reg), TMP1w
}
}
|3:
}
return 1;
}
static int zend_jit_assign_to_typed_ref(dasm_State **Dst,
const zend_op *opline,
uint8_t val_type,
zend_jit_addr val_addr,
zend_jit_addr res_addr,
bool check_exception)
{
| // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
| cbnz TMP1, >2
|.cold_code
|2:
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2x, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, REG0
}
if (!res_addr) {
if (val_type == IS_CONST) {
| EXT_CALL zend_jit_assign_const_to_typed_ref, REG0
} else if (val_type == IS_TMP_VAR) {
| EXT_CALL zend_jit_assign_tmp_to_typed_ref, REG0
} else if (val_type == IS_VAR) {
| EXT_CALL zend_jit_assign_var_to_typed_ref, REG0
} else if (val_type == IS_CV) {
| EXT_CALL zend_jit_assign_cv_to_typed_ref, REG0
} else {
ZEND_UNREACHABLE();
}
} else {
| LOAD_ZVAL_ADDR CARG3, res_addr
if (val_type == IS_CONST) {
| EXT_CALL zend_jit_assign_const_to_typed_ref2, REG0
} else if (val_type == IS_TMP_VAR) {
| EXT_CALL zend_jit_assign_tmp_to_typed_ref2, REG0
} else if (val_type == IS_VAR) {
| EXT_CALL zend_jit_assign_var_to_typed_ref2, REG0
} else if (val_type == IS_CV) {
| EXT_CALL zend_jit_assign_cv_to_typed_ref2, REG0
} else {
ZEND_UNREACHABLE();
}
}
if (check_exception) {
| // if (UNEXPECTED(EG(exception) != NULL)) {
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbz REG0, >8 // END OF zend_jit_assign_to_variable()
| b ->exception_handler
} else {
| b >8
}
|.code
return 1;
}
static int zend_jit_assign_to_variable_call(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr __var_use_addr,
zend_jit_addr var_addr,
uint32_t __var_info,
uint32_t __var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr __res_addr,
bool __check_exception)
{
if (val_info & MAY_BE_UNDEF) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr, ZREG_TMP1
} else {
| IF_ZVAL_TYPE val_addr, IS_UNDEF, >1, ZREG_TMP1
|.cold_code
|1:
ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP);
if (Z_REG(var_addr) != ZREG_FP) {
| str Rx(Z_REG(var_addr)), T1 // save
}
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, Z_OFFSET(val_addr)
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (Z_REG(var_addr) != ZREG_FP) {
| ldr Rx(Z_REG(var_addr)), T1 // restore
}
if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, var_addr
}
| LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
| bl ->assign_const
| b >9
|.code
|1:
}
}
if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, var_addr
}
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2x, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, REG0
}
if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| bl ->assign_tmp
} else if (val_type == IS_CONST) {
| bl ->assign_const
} else if (val_type == IS_TMP_VAR) {
| bl ->assign_tmp
} else if (val_type == IS_VAR) {
if (!(val_info & MAY_BE_REF)) {
| bl ->assign_tmp
} else {
| bl ->assign_var
}
} else if (val_type == IS_CV) {
if (!(val_info & MAY_BE_REF)) {
| bl ->assign_cv_noref
} else {
| bl ->assign_cv
}
if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
|9:
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_assign_to_variable(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
bool check_exception)
/* Labels: 1,2,3,4,5,8 */
{
int done = 0;
zend_reg ref_reg, tmp_reg;
if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_REG0) {
ref_reg = ZREG_FCARG1;
tmp_reg = ZREG_REG0;
} else {
/* ASSIGN_DIM */
ref_reg = ZREG_REG0;
tmp_reg = ZREG_FCARG1;
}
if (var_info & MAY_BE_REF) {
if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) {
| LOAD_ZVAL_ADDR Rx(ref_reg), var_use_addr
var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0);
}
| // if (Z_ISREF_P(variable_ptr)) {
| IF_NOT_Z_TYPE Rx(ref_reg), IS_REFERENCE, >3, TMP1w
| // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
| GET_Z_PTR FCARG1x, Rx(ref_reg)
if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) {
return 0;
}
| add Rx(ref_reg), FCARG1x, #offsetof(zend_reference, val)
|3:
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (RC_MAY_BE_1(var_info)) {
int in_cold = 0;
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_ZVAL_REFCOUNTED var_use_addr, >1, ZREG_TMP1, ZREG_TMP2
|.cold_code
|1:
in_cold = 1;
}
if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_REG0) {
bool keep_gc = 0;
| GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1
#if 0
// TODO: This optiization doesn't work on ARM
if (tmp_reg == ZREG_FCARG1) {
if (Z_MODE(val_addr) == IS_REG) {
keep_gc = 1;
} else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) {
keep_gc = 1;
} else if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
zval *zv = Z_ZV(val_addr);
if (Z_TYPE_P(zv) == IS_DOUBLE) {
if (Z_DVAL_P(zv) == 0) {
keep_gc = 1;
}
} else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
keep_gc = 1;
}
} else if (Z_MODE(val_addr) == IS_MEM_ZVAL) {
if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
keep_gc = 1;
}
}
}
#endif
if (!keep_gc) {
| str Rx(tmp_reg), T1 // save
}
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) {
return 0;
}
if (!keep_gc) {
| ldr FCARG1x, T1 // restore
}
} else {
| GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) {
return 0;
}
}
| GC_DELREF FCARG1x, TMP1w
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
| bne >4
} else {
| bne >8
}
| ZVAL_DTOR_FUNC var_info, opline, TMP1
if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) {
if (check_exception && !(val_info & MAY_BE_UNDEF)) {
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbz REG0, >8
| b ->exception_handler
} else {
| b >8
}
}
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
|4:
| IF_GC_MAY_NOT_LEAK FCARG1x, >8, TMP1w, TMP2w
| EXT_CALL gc_possible_root, REG0
if (in_cold) {
| b >8
}
}
if (check_exception && (val_info & MAY_BE_UNDEF)) {
|8:
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbz REG0, >8
| b ->exception_handler
}
if (in_cold) {
|.code
} else {
done = 1;
}
} else /* if (RC_MAY_BE_N(var_info)) */ {
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5, ZREG_TMP1, ZREG_TMP2
}
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
if (Z_REG(var_use_addr) != ZREG_FP) {
| str Rx(Z_REG(var_use_addr)), T1 // save
}
| GET_ZVAL_PTR FCARG1x, var_use_addr, TMP1
| GC_DELREF FCARG1x, TMP1w
| IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w
| EXT_CALL gc_possible_root, TMP1
if (Z_REG(var_use_addr) != ZREG_FP) {
| ldr Rx(Z_REG(var_use_addr)), T1 // restore
}
} else {
| GET_ZVAL_PTR Rx(tmp_reg), var_use_addr, TMP1
| GC_DELREF Rx(tmp_reg), TMP1w
}
|5:
}
}
if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) {
return 0;
}
|8:
return 1;
}
static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw)
{
zend_jit_addr op2_addr, op3_addr, res_addr;
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
op3_addr = OP1_DATA_ADDR();
if (opline->result_type == IS_UNUSED) {
res_addr = 0;
} else {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr, ZREG_TMP1
val_info &= ~MAY_BE_UNDEF;
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w
| GET_Z_PTR FCARG2x, FCARG1x
| ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))]
| cmp TMP1w, #IS_ARRAY
| bne >2
| add FCARG1x, FCARG2x, #offsetof(zend_reference, val)
| b >3
|.cold_code
|2:
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_jit_prepare_assign_dim_ref, REG0
| mov FCARG1x, RETVALx
| cbnz FCARG1x, >1
| b ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
| bgt >7
}
| // ZVAL_ARR(container, zend_new_array(8));
if (Z_REG(op1_addr) != ZREG_FP) {
| str Rx(Z_REG(op1_addr)), T1 // save
}
| EXT_CALL _zend_new_array_0, REG0
| mov REG0, RETVALx
if (Z_REG(op1_addr) != ZREG_FP) {
| ldr Rx(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
| mov FCARG1x, REG0
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|6:
if (opline->op2_type == IS_UNUSED) {
uint32_t var_info = MAY_BE_NULL;
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, REG0
| // if (UNEXPECTED(!var_ptr)) {
| mov REG0, RETVALx
| cbz REG0, >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| b >9
|.code
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) {
return 0;
}
} else {
uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) {
return 0;
}
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
|8:
| // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
if (opline->op1_type == IS_VAR) {
ZEND_ASSERT(opline->result_type == IS_UNUSED);
if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
}
}
}
if (((op1_info & MAY_BE_ARRAY) &&
(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) ||
(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) &&
(op1_info & MAY_BE_ARRAY)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
| bgt >2
}
| // ZVAL_ARR(container, zend_new_array(8));
if (Z_REG(op1_addr) != ZREG_FP) {
| str Rx(Z_REG(op1_addr)), T1 // save
}
| EXT_CALL _zend_new_array_0, REG0
| mov REG0, RETVALx
if (Z_REG(op1_addr) != ZREG_FP) {
| ldr Rx(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
| mov FCARG1x, REG0
| // ZEND_VM_C_GOTO(assign_dim_op_new_array);
| b <6
|2:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| SET_EX_OPLINE opline, REG0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| mov FCARG2x, xzr
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
if (opline->result_type == IS_UNUSED) {
| mov CARG4, xzr
} else {
| LOAD_ZVAL_ADDR CARG4, res_addr
}
| LOAD_ZVAL_ADDR CARG3, op3_addr
| EXT_CALL zend_jit_assign_dim_helper, REG0
#ifdef ZEND_JIT_USE_RC_INFERENCE
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
/* ASSIGN_DIM may increase refcount of the value */
val_info |= MAY_BE_RCN;
}
#endif
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| b >9 // END
}
|.code
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
|9:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw)
{
zend_jit_addr op2_addr, op3_addr, var_addr;
const void *not_found_exit_addr = NULL;
uint32_t var_info = MAY_BE_NULL;
ZEND_ASSERT(opline->result_type == IS_UNUSED);
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
op3_addr = OP1_DATA_ADDR();
| SET_EX_OPLINE opline, REG0
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w
| GET_Z_PTR FCARG2x, FCARG1x
| ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))]
| cmp TMP1w, #IS_ARRAY
| bne >2
| add FCARG1x, FCARG2x, #offsetof(zend_reference, val)
| b >3
|.cold_code
|2:
| EXT_CALL zend_jit_prepare_assign_dim_ref, REG0
| mov FCARG1x, RETVALx
| cbnz RETVALx, >1
| b ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
| bgt >7
}
if (Z_REG(op1_addr) != ZREG_FP) {
| str Rx(Z_REG(op1_addr)), T1 // save
}
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & MAY_BE_NULL) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
}
| LOAD_32BIT_VAL FCARG1x, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
|1:
}
| // ZVAL_ARR(container, zend_new_array(8));
| EXT_CALL _zend_new_array_0, REG0
| mov REG0, RETVALx
if (Z_REG(op1_addr) != ZREG_FP) {
| ldr Rx(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
| mov FCARG1x, REG0
if (op1_info & MAY_BE_ARRAY) {
| b >1
|.code
|1:
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
|6:
if (opline->op2_type == IS_UNUSED) {
var_info = MAY_BE_NULL;
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, REG0
| mov REG0, RETVALx
| // if (UNEXPECTED(!var_ptr)) {
| cbz REG0, >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| b >9
|.code
} else {
var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
if (dim_type != IS_UNKNOWN
&& dim_type != IS_UNDEF
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
&& (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) {
return 0;
}
|8:
if (not_found_exit_addr && dim_type != IS_REFERENCE) {
| IF_NOT_Z_TYPE, REG0, dim_type, &not_found_exit_addr, TMP1w
var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
| IF_NOT_Z_TYPE, REG0, IS_REFERENCE, >1, TMP1w
| GET_Z_PTR FCARG1x, REG0
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
| cbnz TMP1, >2
| add REG0, FCARG1x, #offsetof(zend_reference, val)
|.cold_code
|2:
| LOAD_ZVAL_ADDR FCARG2x, op3_addr
| LOAD_ADDR CARG3, binary_op
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
}
| b >9
|.code
|1:
}
}
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, may_throw)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
(opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info,
op1_data_range,
0, var_addr, var_def_info, var_info, may_throw)) {
return 0;
}
break;
case ZEND_CONCAT:
if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
may_throw)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
binary_op_type binary_op;
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| mov FCARG2x, xzr
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
binary_op = get_binary_op(opline->extended_value);
| LOAD_ZVAL_ADDR CARG3, op3_addr
| LOAD_ADDR CARG4, binary_op
| EXT_CALL zend_jit_assign_dim_op_helper, REG0
|9:
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception(Dst);
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
| b >9 // END
|.code
|9:
}
} else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY))
&& (!not_found_exit_addr || (var_info & MAY_BE_REF))) {
|.cold_code
|9:
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline, ZREG_TMP1, ZREG_TMP2
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception(Dst);
}
| b >9
|.code
|9:
}
return 1;
}
static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
if (op1_info & MAY_BE_REF) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE, FCARG1x, IS_REFERENCE, >1, TMP1w
| GET_Z_PTR FCARG1x, FCARG1x
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
| cbnz TMP1, >2
| add FCARG1x, FCARG1x, #offsetof(zend_reference, val)
|.cold_code
|2:
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
| LOAD_ADDR CARG3, binary_op
| SET_EX_OPLINE opline, REG0
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
}
zend_jit_check_exception(Dst);
| b >9
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
int result;
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw);
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
result = zend_jit_long_math_helper(Dst, opline, opline->extended_value,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw);
break;
case ZEND_CONCAT:
result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw);
break;
default:
ZEND_UNREACHABLE();
}
|9:
return result;
}
static int zend_jit_cmp_long_long(dasm_State **Dst,
const zend_op *opline,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
bool swap = 0;
bool result;
if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
if (!smart_branch_opcode ||
smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
| SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE), TMP1w, TMP2
}
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
if (!result) {
| b => target_label
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
if (result) {
| b => target_label
}
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
if (skip_comparison) {
if (Z_MODE(op1_addr) != IS_REG &&
(Z_MODE(op2_addr) == IS_REG ||
(Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) {
swap = 1;
}
} else if (Z_MODE(op1_addr) == IS_REG) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
| cmp Rx(Z_REG(op1_addr)), xzr
} else {
| LONG_CMP Z_REG(op1_addr), op2_addr, TMP1
}
} else if (Z_MODE(op2_addr) == IS_REG) {
if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) {
| cmp Rx(Z_REG(op2_addr)), xzr
} else {
| LONG_CMP Z_REG(op2_addr), op1_addr, TMP1
}
swap = 1;
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) {
| LONG_CMP_WITH_CONST op2_addr, Z_LVAL_P(Z_ZV(op1_addr)), TMP1, TMP2
swap = 1;
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) {
| LONG_CMP_WITH_CONST op1_addr, Z_LVAL_P(Z_ZV(op2_addr)), TMP1, TMP2
} else {
| GET_ZVAL_LVAL ZREG_REG0, op1_addr, TMP1
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
| cmp Rx(ZREG_REG0), xzr
} else {
| LONG_CMP ZREG_REG0, op2_addr, TMP1
}
}
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| cset REG0w, eq
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| cset REG0w, ne
break;
case ZEND_IS_SMALLER:
if (swap) {
| cset REG0w, gt
} else {
| cset REG0w, lt
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| cset REG0w, ge
} else {
| cset REG0w, le
}
break;
default:
ZEND_UNREACHABLE();
}
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| bne &exit_addr
} else {
| bne => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| beq &exit_addr
} else {
| beq => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| bne &exit_addr
} else {
| beq => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| ble &exit_addr
} else {
| ble => target_label
}
} else {
if (exit_addr) {
| bge &exit_addr
} else {
| bge => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| blt &exit_addr
} else {
| blt => target_label
}
} else {
if (exit_addr) {
| bgt &exit_addr
} else {
| bgt => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| beq &exit_addr
} else {
| beq => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| bne &exit_addr
} else {
| bne => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| beq &exit_addr
} else {
| bne => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| bgt &exit_addr
} else {
| bgt => target_label
}
} else {
if (exit_addr) {
| blt &exit_addr
} else {
| blt => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| bge &exit_addr
} else {
| bge => target_label
}
} else {
if (exit_addr) {
| ble &exit_addr
} else {
| ble => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| cset REG0w, eq
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| cset REG0w, ne
break;
case ZEND_IS_SMALLER:
if (swap) {
| cset REG0w, gt
} else {
| cset REG0w, lt
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| cset REG0w, ge
} else {
| cset REG0w, le
}
break;
default:
ZEND_UNREACHABLE();
}
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
return 1;
}
static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| bne &exit_addr
} else {
| bne => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
| bvs >1
if (exit_addr) {
| beq &exit_addr
} else {
| beq => target_label
}
|1:
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| bvs &exit_addr
| bne &exit_addr
} else {
| bvs >1
| beq => target_label
|1:
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| bvs &exit_addr
| bls &exit_addr
} else {
| bvs => target_label
| bls => target_label
}
} else {
if (exit_addr) {
| bhs &exit_addr
} else {
| bhs => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| bvs &exit_addr
| blo &exit_addr
} else {
| bvs => target_label
| blo => target_label
}
} else {
if (exit_addr) {
| bhi &exit_addr
} else {
| bhi => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| bvs >1
if (exit_addr) {
| beq &exit_addr
} else {
| beq => target_label
}
|1:
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| bne &exit_addr
} else {
| bne => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| bvs >1
| beq &exit_addr
|1:
} else {
| bne => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
| bvs >1 // Always False if involving NaN
if (exit_addr) {
| bhi &exit_addr
} else {
| bhi => target_label
}
|1:
} else {
| bvs >1
if (exit_addr) {
| blo &exit_addr
} else {
| blo => target_label
}
|1:
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| bvs >1 // Always False if involving NaN
if (exit_addr) {
| bhs &exit_addr
} else {
| bhs => target_label
}
|1:
} else {
| bvs >1
if (exit_addr) {
| bls &exit_addr
} else {
| bls => target_label
}
|1:
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| bne => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| bvs >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| beq => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
break;
case ZEND_IS_SMALLER:
if (swap) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| bvs => target_label
| bls => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| bhs => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| bvs => target_label
| blo => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| bhi => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| bvs >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
| beq => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
| bvs => target_label
| bne => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
break;
case ZEND_IS_SMALLER:
if (swap) {
| cset REG0w, hi
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
| bvs >1 // Always False if involving NaN
| bhi => target_label
|1:
} else {
| bvs >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
| blo => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| cset REG0w, hs
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
| bvs >1 // Always False if involving NaN
| bhs => target_label
|1:
} else {
| bvs >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
| bls => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
}
break;
default:
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| bvs >1
| mov REG0, #IS_TRUE
| beq >2
|1:
| mov REG0, #IS_FALSE
|2:
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| bvs >1
| mov REG0, #IS_FALSE
| beq >2
|1:
| mov REG0, #IS_TRUE
|2:
break;
case ZEND_IS_SMALLER:
| bvs >1
| mov REG0, #IS_TRUE
|| if (swap) {
| bhi >2
|| } else {
| blo >2
|| }
|1:
| mov REG0, #IS_FALSE
|2:
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| bvs >1
| mov REG0, #IS_TRUE
|| if (swap) {
| bhs >2
|| } else {
| bls >2
|| }
|1:
| mov REG0, #IS_FALSE
|2:
break;
default:
ZEND_UNREACHABLE();
}
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
return 1;
}
static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_FPR0;
| DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_REG0, ZREG_TMP1
| DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP
return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_FPR0;
| DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_REG0, ZREG_TMP1
| DOUBLE_CMP tmp_reg, op1_addr, ZREG_TMP1, ZREG_FPTMP
return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
bool swap = 0;
if (Z_MODE(op1_addr) == IS_REG) {
| DOUBLE_CMP Z_REG(op1_addr), op2_addr, ZREG_TMP1, ZREG_FPTMP
} else if (Z_MODE(op2_addr) == IS_REG) {
| DOUBLE_CMP Z_REG(op2_addr), op1_addr, ZREG_TMP1, ZREG_FPTMP
swap = 1;
} else {
zend_reg tmp_reg = ZREG_FPR0;
| GET_ZVAL_DVAL tmp_reg, op1_addr, ZREG_TMP1
| DOUBLE_CMP tmp_reg, op2_addr, ZREG_TMP1, ZREG_FPTMP
}
return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
| tst RETVALw, RETVALw
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
| cset REG0w, eq
break;
case ZEND_IS_NOT_EQUAL:
| cset REG0w, ne
break;
case ZEND_IS_SMALLER:
| cset REG0w, lt
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| cset REG0w, le
break;
default:
ZEND_UNREACHABLE();
}
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
if (exit_addr) {
| bne &exit_addr
} else {
| bne => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| beq &exit_addr
} else {
| beq => target_label
}
break;
case ZEND_IS_SMALLER:
if (exit_addr) {
| bge &exit_addr
} else {
| bge => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (exit_addr) {
| bgt &exit_addr
} else {
| bgt => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
if (exit_addr) {
| beq &exit_addr
} else {
| beq => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| bne &exit_addr
} else {
| bne => target_label
}
break;
case ZEND_IS_SMALLER:
if (exit_addr) {
| blt &exit_addr
} else {
| blt => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (exit_addr) {
| ble &exit_addr
} else {
| ble => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
| cset REG0w, eq
break;
case ZEND_IS_NOT_EQUAL:
| cset REG0w, ne
break;
case ZEND_IS_SMALLER:
| cset REG0w, lt
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| cset REG0w, le
break;
default:
ZEND_UNREACHABLE();
}
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
return 1;
}
static int zend_jit_cmp(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var);
bool has_slow;
has_slow =
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1
}
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if (op2_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3, ZREG_TMP1
|.cold_code
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
}
if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| b >6
|.code
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1
}
}
if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
|4:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| b >6
}
if (!same_ops) {
|5:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1
}
if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| b >6
}
|.code
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops && (op2_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & MAY_BE_DOUBLE) {
|.cold_code
}
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9, ZREG_TMP1
}
if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op2_info & MAY_BE_DOUBLE) {
| b >6
|.code
}
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9, ZREG_TMP1
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops && (op1_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9, ZREG_TMP1
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
}
|3:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9, ZREG_TMP1
}
if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
| b >6
|.code
}
}
}
if (has_slow ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
if (has_slow) {
|.cold_code
|9:
}
| SET_EX_OPLINE opline, REG0
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
| IF_NOT_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w
| LOAD_32BIT_VAL FCARG1x, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
| LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
|1:
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1
| str FCARG1x, T1 // save
| LOAD_32BIT_VAL FCARG1x, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
| ldr FCARG1x, T1 // restore
| LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
| b >2
|1:
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
|2:
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
| EXT_CALL zend_compare, REG0
if ((opline->opcode != ZEND_CASE &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
| str RETVALw, T1 // save
if (opline->opcode != ZEND_CASE) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL, ZREG_TMP1, ZREG_TMP2
| ldr RETVALw, T1 // restore
}
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (has_slow) {
| b >6
|.code
}
}
|6:
return 1;
}
static int zend_jit_identical(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
uint32_t identical_label = (uint32_t)-1;
uint32_t not_identical_label = (uint32_t)-1;
if (smart_branch_opcode && !exit_addr) {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
if (smart_branch_opcode == ZEND_JMPZ) {
not_identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
identical_label = target_label;
} else {
ZEND_UNREACHABLE();
}
} else {
if (smart_branch_opcode == ZEND_JMPZ) {
identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
not_identical_label = target_label;
} else {
ZEND_UNREACHABLE();
}
}
}
if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
return 0;
}
return 1;
} else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
return 1;
}
if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) {
op1_info |= MAY_BE_NULL;
op2_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
| b >1
|.code
|1:
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
| IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, REG0
| str FCARG1x, T1 // save
| LOAD_32BIT_VAL FCARG1w, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| ldr FCARG1x, T1 // restore
| LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
| b >1
|.code
|1:
} else if (op1_info & MAY_BE_UNDEF) {
op1_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_Z_TYPE FCARG1x, IS_UNDEF, >1, TMP1w
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
| b >1
|.code
|1:
if (opline->op2_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
} else if (op2_info & MAY_BE_UNDEF) {
op2_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
| IF_Z_TYPE FCARG2x, IS_UNDEF, >1, TMP1w
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
| b >1
|.code
|1:
if (opline->op1_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
} else if ((op1_info & op2_info & MAY_BE_ANY) != 0) {
if (opline->op1_type != IS_CONST) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
}
if (opline->op2_type != IS_CONST) {
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
if (opline->op1_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
}
if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
if ((opline->opcode != ZEND_CASE_STRICT &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
if (opline->opcode != ZEND_CASE_STRICT) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
if (smart_branch_opcode) {
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| b =>not_identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2
if (may_throw) {
zend_jit_check_exception(Dst);
}
}
return 1;
}
if (opline->op1_type & (IS_CV|IS_VAR)) {
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
}
if (opline->op2_type & (IS_CV|IS_VAR)) {
| ZVAL_DEREF FCARG2x, op2_info, TMP1w
}
if (has_concrete_type(op1_info)
&& has_concrete_type(op2_info)
&& concrete_type(op1_info) == concrete_type(op2_info)
&& concrete_type(op1_info) <= IS_TRUE) {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
}
} else if (identical_label != (uint32_t)-1) {
| b =>identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2
}
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
}
} else if (identical_label != (uint32_t)-1) {
| b =>identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE), TMP1w, TMP2
}
} else {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| b =>not_identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE), TMP1w, TMP2
}
}
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op1_addr);
| ldrb TMP1w, [FCARG2x, #offsetof(zval, u1.v.type)]
| cmp TMP1w, #Z_TYPE_P(val)
if (smart_branch_opcode) {
if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) {
| bne >8
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
} else if (identical_label != (uint32_t)-1) {
| b =>identical_label
} else {
| b >9
}
|8:
} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| beq &exit_addr
} else if (identical_label != (uint32_t)-1) {
| beq =>identical_label
} else {
| beq >9
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| cset REG0w, eq
} else {
| cset REG0w, ne
}
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b &exit_addr
}
} else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) {
| b =>not_identical_label
}
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op2_addr);
| ldrb TMP1w, [FCARG1x, #offsetof(zval, u1.v.type)]
| cmp TMP1w, #Z_TYPE_P(val)
if (smart_branch_opcode) {
if (opline->opcode != ZEND_CASE_STRICT
&& opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) {
| bne >8
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
} else if (identical_label != (uint32_t)-1) {
| b =>identical_label
} else {
| b >9
}
|8:
} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| beq &exit_addr
} else if (identical_label != (uint32_t)-1) {
| beq =>identical_label
} else {
| beq >9
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| cset REG0w, eq
} else {
| cset REG0w, ne
}
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
if (opline->opcode != ZEND_CASE_STRICT
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
}
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| b =>not_identical_label
}
}
} else {
if (opline->op1_type == IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
if (opline->op2_type == IS_CONST) {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
| EXT_CALL zend_is_identical, REG0
if ((opline->opcode != ZEND_CASE_STRICT &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
| str RETVALw, T1 // save
if (opline->opcode != ZEND_CASE_STRICT) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| ldr RETVALw, T1 // restore
}
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| cbnz RETVALw, &exit_addr
} else {
| cbz RETVALw, &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| cbz RETVALw, =>not_identical_label
if (identical_label != (uint32_t)-1) {
| b =>identical_label
}
} else if (identical_label != (uint32_t)-1) {
| cbnz RETVALw, =>identical_label
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| add RETVALw, RETVALw, #2
} else {
| neg RETVALw, RETVALw
| add RETVALw, RETVALw, #3
}
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, RETVALw, TMP1
}
}
|9:
if (may_throw) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr)
{
uint32_t true_label = -1;
uint32_t false_label = -1;
bool set_bool = 0;
bool set_bool_not = 0;
bool set_delayed = 0;
bool jmp_done = 0;
if (branch_opcode == ZEND_BOOL) {
set_bool = 1;
} else if (branch_opcode == ZEND_BOOL_NOT) {
set_bool = 1;
set_bool_not = 1;
} else if (branch_opcode == ZEND_JMPZ) {
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
} else if (branch_opcode == ZEND_JMPZ_EX) {
set_bool = 1;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ_EX) {
set_bool = 1;
true_label = target_label;
} else {
ZEND_UNREACHABLE();
}
if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
if (zend_is_true(Z_ZV(op1_addr))) {
/* Always TRUE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
}
}
if (true_label != (uint32_t)-1) {
| b =>true_label
}
} else {
/* Always FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
}
}
if (false_label != (uint32_t)-1) {
| b =>false_label
}
}
return 1;
}
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
/* Always TRUE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
}
}
if (true_label != (uint32_t)-1) {
| b =>true_label
}
} else {
if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
/* Always FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
}
}
} else {
| CMP_ZVAL_TYPE op1_addr, IS_TRUE, ZREG_TMP1
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if ((op1_info & MAY_BE_LONG) &&
!(op1_info & MAY_BE_UNDEF) &&
!set_bool) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ) {
| blt >9
} else {
| blt &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| blt =>false_label
} else {
| blt >9
}
jmp_done = 1;
} else {
| bgt >2
}
}
if (!(op1_info & MAY_BE_TRUE)) {
/* It's FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
}
}
} else {
if (exit_addr) {
if (set_bool) {
| bne >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| b &exit_addr
} else {
| b >9
}
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| bne &exit_addr
}
}
} else {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| beq &exit_addr
} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| bne &exit_addr
} else {
| beq >9
}
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (set_bool) {
| bne >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
if (true_label != (uint32_t)-1) {
| b =>true_label
} else {
| b >9
}
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
} else {
if (true_label != (uint32_t)-1) {
| beq =>true_label
} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| bne =>false_label
jmp_done = 1;
} else {
| beq >9
}
}
} else if (set_bool) {
| cset REG0w, eq
if (set_bool_not) {
| neg REG0w, REG0w
| add REG0w, REG0w, #3
} else {
| add REG0w, REG0w, #2
}
if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) {
set_delayed = 1;
} else {
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
}
}
}
/* It's FALSE, but may be UNDEF */
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & MAY_BE_ANY) {
if (set_delayed) {
| CMP_ZVAL_TYPE op1_addr, IS_UNDEF, ZREG_TMP1
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
| beq >1
} else {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
}
|.cold_code
|1:
}
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
| b &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| b =>false_label
}
if (op1_info & MAY_BE_ANY) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| b >9
}
} else if (false_label == (uint32_t)-1) {
| b >9
}
|.code
}
}
if (!jmp_done) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
if (op1_info & MAY_BE_LONG) {
| b >9
}
} else if (op1_info & MAY_BE_LONG) {
| b &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| b =>false_label
} else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| b >9
}
}
}
}
if (op1_info & MAY_BE_LONG) {
|2:
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2, ZREG_TMP1
}
if (Z_MODE(op1_addr) == IS_REG) {
| tst Rx(Z_REG(op1_addr)), Rx(Z_REG(op1_addr))
} else {
| LONG_CMP_WITH_CONST op1_addr, Z_L(0), TMP1, TMP2
}
if (set_bool) {
| cset REG0w, ne
if (set_bool_not) {
| neg REG0w, REG0w
| add REG0w, REG0w, #3
} else {
| add REG0w, REG0w, #2
}
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| bne &exit_addr
} else {
| beq &exit_addr
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (true_label != (uint32_t)-1) {
| bne =>true_label
if (false_label != (uint32_t)-1) {
| b =>false_label
}
} else {
| beq =>false_label
}
}
}
if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.cold_code
}
|2:
| fmov FPR0, xzr // TODO: "movi d0, #0" is not recognized by DynASM/arm64
| DOUBLE_CMP ZREG_FPR0, op1_addr, ZREG_TMP1, ZREG_FPTMP
if (set_bool) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
| bvs &exit_addr
| bne &exit_addr
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
} else {
| bvs >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| beq &exit_addr
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
}
} else if (false_label != (uint32_t)-1) { // JMPZ_EX
| bvs >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| beq => false_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
} else if (true_label != (uint32_t)-1) { // JMPNZ_EX
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
| bvs => true_label
| bne => true_label
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
} else if (set_bool_not) { // BOOL_NOT
| mov REG0w, #IS_FALSE
| bvs >1
| bne >1
| mov REG0w, #IS_TRUE
|1:
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
} else { // BOOL
| mov REG0w, #IS_TRUE
| bvs >1
| bne >1
| mov REG0w, #IS_FALSE
|1:
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
|.code
}
} else {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| bvs &exit_addr
| bne &exit_addr
|1:
} else {
| bvs >1
| beq &exit_addr
|1:
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
} else {
ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1);
if (false_label != (uint32_t)-1 ) {
| bvs >1
| beq => false_label
|1:
if (true_label != (uint32_t)-1) {
| b =>true_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
} else {
| bvs => true_label
| bne => true_label
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.code
}
}
} else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.cold_code
|2:
}
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_is_true, REG0
| mov REG0, RETVALx
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, >3, ZREG_TMP1, ZREG_TMP2
}
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
| GC_DELREF FCARG1x, TMP1w
| bne >3
// In x86, r0 is used in macro ZVAL_DTOR_FUNC as temporary register, hence, r0 should be saved/restored
// before/after this macro. In AArch64, TMP1 is used, but we still have to store REG0,
// because it's clobbered by function call.
| str REG0, T1 // save
| ZVAL_DTOR_FUNC op1_info, opline, TMP1
| ldr REG0, T1 // restore
|3:
}
if (may_throw) {
| MEM_LOAD_64_ZTS ldr, REG1, executor_globals, exception, TMP1
| cbnz REG1, ->exception_handler_undef
}
if (set_bool) {
if (set_bool_not) {
| neg REG0w, REG0w
| add REG0w, REG0w, #3
} else {
| add REG0w, REG0w, #2
}
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
if (exit_addr) {
| CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| bne &exit_addr
} else {
| beq &exit_addr
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
| CMP_ZVAL_TYPE res_addr, IS_FALSE, ZREG_TMP1
if (true_label != (uint32_t)-1) {
| bne =>true_label
if (false_label != (uint32_t)-1) {
| b =>false_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
} else {
| beq =>false_label
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
|.code
}
} else {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| cbnz REG0w, &exit_addr
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
} else {
| cbz REG0w, &exit_addr
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
}
} else if (true_label != (uint32_t)-1) {
| cbnz REG0w, =>true_label
if (false_label != (uint32_t)-1) {
| b =>false_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
} else {
| cbz REG0w, =>false_label
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| b >9
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.code
}
}
}
|9:
return 1;
}
static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
{
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr = op1_def_addr;
}
}
if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
if (op1_info & MAY_BE_UNDEF) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT(opline->op1_type == IS_CV);
if (op2_addr != op2_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
return 0;
}
if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
op2_addr = op2_def_addr;
}
}
if (Z_MODE(op1_addr) != IS_REG
&& Z_MODE(op1_use_addr) == IS_REG
&& !Z_LOAD(op1_use_addr)
&& !Z_STORE(op1_use_addr)) {
/* Force type update */
op1_info |= MAY_BE_UNDEF;
}
if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr,
may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
/* copy of hidden zend_closure */
typedef struct _zend_closure {
zend_object std;
zend_function func;
zval this_ptr;
zend_class_entry *called_scope;
zif_handler orig_internal_handler;
} zend_closure;
static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| // Check Stack Overflow
| MEM_LOAD_64_ZTS ldr, REG1, executor_globals, vm_stack_end, TMP1
| MEM_LOAD_OP_ZTS sub, ldr, REG1, executor_globals, vm_stack_top, TMP1, TMP2
| CMP_64_WITH_CONST_32 REG1, used_stack, TMP1
| blo &exit_addr
return 1;
}
static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack)
{
uint32_t used_stack;
bool stack_check = 1;
// REG0 -> zend_function
// FCARG1 -> used_stack
if (func) {
used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
if ((int)used_stack <= checked_stack) {
stack_check = 0;
}
} else {
used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval);
| // if (EXPECTED(ZEND_USER_CODE(func->type))) {
if (!is_closure) {
| LOAD_32BIT_VAL FCARG1w, used_stack
| // Check whether REG0 is an internal function.
| ldrb TMP1w, [REG0, #offsetof(zend_function, type)]
| TST_32_WITH_CONST TMP1w, 1, TMP2w
| bne >1
} else {
| LOAD_32BIT_VAL FCARG1w, used_stack
}
| // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
| LOAD_32BIT_VAL REG2w, opline->extended_value
if (!is_closure) {
| ldr TMP1w, [REG0, #offsetof(zend_function, op_array.num_args)]
| cmp REG2w, TMP1w
| csel REG2w, REG2w, TMP1w, le
| ldr TMP1w, [REG0, #offsetof(zend_function, op_array.last_var)]
| sub REG2w, REG2w, TMP1w
| ldr TMP1w, [REG0, #offsetof(zend_function, op_array.T)]
| sub REG2w, REG2w, TMP1w
} else {
| ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.num_args)]
| cmp REG2w, TMP1w
| csel REG2w, REG2w, TMP1w, le
| ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.last_var)]
| sub REG2w, REG2w, TMP1w
| ldr TMP1w, [REG0, #offsetof(zend_closure, func.op_array.T)]
| sub REG2w, REG2w, TMP1w
}
| sxtw REG2, REG2w
| sub FCARG1x, FCARG1x, REG2, lsl #4
|1:
}
zend_jit_start_reuse_ip();
| // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
| MEM_LOAD_64_ZTS ldr, RX, executor_globals, vm_stack_top, TMP1
if (stack_check) {
| // Check Stack Overflow
| MEM_LOAD_64_ZTS ldr, REG2, executor_globals, vm_stack_end, TMP1
| sub REG2, REG2, RX
if (func) {
| CMP_64_WITH_CONST_32 REG2, used_stack, TMP1
} else {
| cmp REG2, FCARG1x
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| blo &exit_addr
} else {
| blo >1
| // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
|.cold_code
|1:
if (func) {
| LOAD_32BIT_VAL FCARG1w, used_stack
}
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_jit_int_extend_stack_helper, REG0
} else {
if (!is_closure) {
| mov FCARG2x, REG0
} else {
| add FCARG2x, REG0, #offsetof(zend_closure, func)
}
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_jit_extend_stack_helper, REG0
}
| mov RX, RETVALx
| b >1
|.code
}
}
if (func) {
|| if (arm64_may_encode_imm12((int64_t)used_stack)) {
| MEM_UPDATE_ZTS add, ldr, str, #used_stack, executor_globals, vm_stack_top, REG2, TMP1
|| } else {
| LOAD_32BIT_VAL TMP1w, used_stack
| MEM_UPDATE_ZTS add, ldr, str, TMP1, executor_globals, vm_stack_top, REG2, TMP2
|| }
} else {
| MEM_UPDATE_ZTS add, ldr, str, FCARG1x, executor_globals, vm_stack_top, REG2, TMP1
}
| // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
| // ZEND_SET_CALL_INFO(call, 0, call_info);
| LOAD_32BIT_VAL TMP1w, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION)
| str TMP1w, EX:RX->This.u1.type_info
}
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
| // call->func = func;
|1:
| ADDR_STORE EX:RX->func, func, REG1
} else {
if (!is_closure) {
| // call->func = func;
| str REG0, EX:RX->func
} else {
| // call->func = &closure->func;
| add REG1, REG0, #offsetof(zend_closure, func)
| str REG1, EX:RX->func
}
|1:
}
if (opline->opcode == ZEND_INIT_METHOD_CALL) {
| // Z_PTR(call->This) = obj;
| ldr REG1, T1
| str REG1, EX:RX->This.value.ptr
if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
| // call->call_info |= ZEND_CALL_HAS_THIS;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| LOAD_32BIT_VAL TMP1w, ZEND_CALL_HAS_THIS
| str TMP1w, EX:RX->This.u1.type_info
} else {
| ldr TMP1w, EX:RX->This.u1.type_info
| BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_HAS_THIS, TMP2w
| str TMP1w, EX:RX->This.u1.type_info
}
} else {
if (opline->op1_type == IS_CV) {
| // GC_ADDREF(obj);
| GC_ADDREF REG1, TMP1w
}
| // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| LOAD_32BIT_VAL TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)
| str TMP1w, EX:RX->This.u1.type_info
} else {
| ldr TMP1w, EX:RX->This.u1.type_info
| BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS), TMP2w
| str TMP1w, EX:RX->This.u1.type_info
}
}
} else if (!is_closure) {
| // Z_CE(call->This) = called_scope;
| str xzr, EX:RX->This.value.ptr
} else {
if (opline->op2_type == IS_CV) {
| // GC_ADDREF(closure);
| GC_ADDREF REG0, TMP1w
}
| // object_or_called_scope = closure->called_scope;
| ldr REG1, [REG0, #offsetof(zend_closure, called_scope)]
| str REG1, EX:RX->This.value.ptr
| // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
| // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
| ldr REG2w, [REG0, #offsetof(zend_closure, func.common.fn_flags)]
| BW_OP_32_WITH_CONST and, REG2w, REG2w, ZEND_ACC_FAKE_CLOSURE, TMP1w
| BW_OP_32_WITH_CONST orr, REG2w, REG2w, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE), TMP1w
| // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
| ldrb TMP1w, [REG0, #offsetof(zend_closure, this_ptr.u1.v.type)]
| cmp TMP1w, #IS_UNDEF
| beq >1
| // call_info |= ZEND_CALL_HAS_THIS;
| BW_OP_32_WITH_CONST orr, REG2w, REG2w, ZEND_CALL_HAS_THIS, TMP1w
| // object_or_called_scope = Z_OBJ(closure->this_ptr);
| ldr REG1, [REG0, #offsetof(zend_closure, this_ptr.value.ptr)]
|1:
| // ZEND_SET_CALL_INFO(call, 0, call_info);
| ldr TMP1w, EX:RX->This.u1.type_info
| orr TMP1w, TMP1w, REG2w
| str TMP1w, EX:RX->This.u1.type_info
| // Z_PTR(call->This) = object_or_called_scope;
| str REG1, EX:RX->This.value.ptr
| ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.run_time_cache__ptr)]
| cbnz TMP1, >1
| add FCARG1x, REG0, #offsetof(zend_closure, func)
| EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0
|1:
}
| // ZEND_CALL_NUM_ARGS(call) = num_args;
| LOAD_32BIT_VAL TMP1w, opline->extended_value
| str TMP1w, EX:RX->This.u2.num_args
return 1;
}
static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline)
{
int32_t exit_point;
const void *exit_addr;
if (func->type == ZEND_INTERNAL_FUNCTION) {
#ifdef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
return 0;
#endif
} else if (func->type == ZEND_USER_FUNCTION) {
if (!zend_accel_in_shm(func->op_array.opcodes)) {
/* op_array and op_array->opcodes are not persistent. We can't link. */
return 0;
}
} else {
ZEND_UNREACHABLE();
return 0;
}
exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| // call = EX(call);
| ldr REG1, EX->call
while (level > 0) {
| ldr REG1, EX:REG1->prev_execute_data
level--;
}
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
| ldr REG1, EX:REG1->func
| LOAD_ADDR REG2, ((ptrdiff_t)opcodes)
| ldr TMP1, [REG1, #offsetof(zend_op_array, opcodes)]
| cmp TMP1, REG2
| bne &exit_addr
} else {
| LOAD_ADDR REG2, ((ptrdiff_t)func)
| ldr TMP1, EX:REG1->func
| cmp TMP1, REG2
| bne &exit_addr
}
return 1;
}
static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (!func
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL) {
func = (zend_function*)trace->func;
}
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_INTERNAL_FUNCTION) {
/* load constant address later */
} else if (func && op_array == &func->op_array) {
/* recursive call */
| ldr REG0, EX->func
} else {
| // if (CACHED_PTR(opline->result.num))
| ldr REG2, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG2, opline->result.num, TMP1
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& func
&& (func->common.fn_flags & ZEND_ACC_IMMUTABLE)
&& opline->opcode != ZEND_INIT_FCALL) {
/* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */
| LOAD_ADDR REG1, ((ptrdiff_t)func)
| cmp REG0, REG1
| bne >1
} else {
| cbz REG0, >1
}
|.cold_code
|1:
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_USER_FUNCTION
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
| LOAD_ADDR FCARG1x, func
| MEM_ACCESS_64_WITH_UOFFSET str, FCARG1x, REG2, opline->result.num, TMP1
| EXT_CALL zend_jit_init_func_run_time_cache_helper, REG0
| mov REG0, RETVALx
| b >3
} else {
zval *zv = RT_CONSTANT(opline, opline->op2);
if (opline->opcode == ZEND_INIT_FCALL) {
| LOAD_ADDR FCARG1x, Z_STR_P(zv);
| ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1
| EXT_CALL zend_jit_find_func_helper, REG0
} else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
| LOAD_ADDR FCARG1x, Z_STR_P(zv + 1);
| ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1
| EXT_CALL zend_jit_find_func_helper, REG0
} else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
| LOAD_ADDR FCARG1x, zv;
| ADD_SUB_64_WITH_CONST_32 add, FCARG2x, REG2, opline->result.num, TMP1
| EXT_CALL zend_jit_find_ns_func_helper, REG0
} else {
ZEND_UNREACHABLE();
}
| // Get the return value of function zend_jit_find_func_helper/zend_jit_find_ns_func_helper
| mov REG0, RETVALx
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline,
func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (!func || opline->opcode == ZEND_INIT_FCALL) {
| cbnz REG0, >3
} else if (func->type == ZEND_USER_FUNCTION
&& !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
const zend_op *opcodes = func->op_array.opcodes;
| LOAD_ADDR REG1, ((ptrdiff_t)opcodes)
| ldr TMP1, [REG0, #offsetof(zend_op_array, opcodes)]
| cmp TMP1, REG1
| beq >3
} else {
| LOAD_ADDR REG1, ((ptrdiff_t)func)
| cmp REG0, REG1
| beq >3
}
| b &exit_addr
} else {
| cbnz REG0, >3
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, REG0
| b ->undefined_function
}
}
|.code
|3:
}
if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) {
return 0;
}
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
return 1;
}
static int zend_jit_init_method_call(dasm_State **Dst,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
zend_jit_trace_rec *trace,
int checked_stack,
bool polymorphic_side_trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
zval *function_name;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
function_name = RT_CONSTANT(opline, opline->op2);
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (polymorphic_side_trace) {
/* function is passed in r0 from parent_trace */
} else {
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
| GET_ZVAL_PTR FCARG1x, this_addr, TMP1
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
} else {
/* Hack: Convert reference to regular value to simplify JIT code */
ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| EXT_CALL zend_jit_unref_helper, REG0
|1:
}
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
|.cold_code
|1:
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| SET_EX_OPLINE opline, REG0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
| EXT_CALL zend_jit_invalid_method_call_tmp, REG0
} else {
| EXT_CALL zend_jit_invalid_method_call, REG0
}
| b ->exception_handler
|.code
}
}
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
}
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
| str FCARG1x, T1 // save
if (func) {
| // fbc = CACHED_PTR(opline->result.num + sizeof(void*));
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1
| cbz REG0, >1
} else {
| // if (CACHED_PTR(opline->result.num) == obj->ce)) {
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->result.num, TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
| cmp REG2, TMP1
| bne >1
| // fbc = CACHED_PTR(opline->result.num + sizeof(void*));
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->result.num + sizeof(void*)), TMP1
}
|.cold_code
|1:
| LOAD_ADDR FCARG2x, function_name
if (TMP_ZVAL_OFFSET == 0) {
| mov CARG3, sp
} else {
| add CARG3, sp, #TMP_ZVAL_OFFSET
}
| SET_EX_OPLINE opline, REG0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
| EXT_CALL zend_jit_find_method_tmp_helper, REG0
} else {
| EXT_CALL zend_jit_find_method_helper, REG0
}
| mov REG0, RETVALx
| cbnz REG0, >2
| b ->exception_handler
|.code
|2:
}
if ((!func || zend_jit_may_be_modified(func, op_array))
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
) {
int32_t exit_point;
const void *exit_addr;
exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
func = (zend_function*)trace->func;
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
| LOAD_ADDR TMP1, opcodes
| ldr TMP2, [REG0, #offsetof(zend_op_array, opcodes)]
| cmp TMP2, TMP1
| bne &exit_addr
} else {
| LOAD_ADDR TMP1, func
| cmp REG0, TMP1
| bne &exit_addr
}
}
if (!func) {
| // if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
| ldr TMP1w, [REG0, #offsetof(zend_function, common.fn_flags)]
| TST_32_WITH_CONST TMP1w, ZEND_ACC_STATIC, TMP2w
| bne >1
|.cold_code
|1:
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
| ldr FCARG1x, T1 // restore
| mov FCARG2x, REG0
| LOAD_32BIT_VAL CARG3w, opline->extended_value
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
| EXT_CALL zend_jit_push_static_metod_call_frame_tmp, REG0
} else {
| EXT_CALL zend_jit_push_static_metod_call_frame, REG0
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
| cbz RETVALx, ->exception_handler
}
| mov RX, RETVALx
}
if (!func) {
| b >9
|.code
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) {
return 0;
}
}
if (!func) {
|9:
}
zend_jit_start_reuse_ip();
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
return 1;
}
static int zend_jit_init_closure_call(dasm_State **Dst,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
zend_jit_trace_rec *trace,
int checked_stack)
{
zend_function *func = NULL;
zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
| GET_ZVAL_PTR REG0, op2_addr, TMP1
if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
&& !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| LOAD_ADDR FCARG1x, ((ptrdiff_t)zend_ce_closure)
| ldr, TMP1, [REG0, #offsetof(zend_object, ce)]
| cmp TMP1, FCARG1x
| bne &exit_addr
if (ssa->var_info && ssa_op->op2_use >= 0) {
ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
}
}
if (trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
&& trace->func->type == ZEND_USER_FUNCTION) {
const zend_op *opcodes;
int32_t exit_point;
const void *exit_addr;
func = (zend_function*)trace->func;
opcodes = func->op_array.opcodes;
exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| LOAD_ADDR FCARG1x, ((ptrdiff_t)opcodes)
| ldr TMP1, [REG0, #offsetof(zend_closure, func.op_array.opcodes)]
| cmp TMP1, FCARG1x
| bne &exit_addr
}
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) {
return 0;
}
if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
return 0;
}
}
return 1;
}
static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
const zend_function *func = NULL;
uint32_t i;
zend_jit_addr res_addr;
uint32_t call_num_args = 0;
bool unknown_num_args = 0;
const void *exit_addr = NULL;
const zend_op *prev_opline;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
} else {
/* CPU stack allocated temporary zval */
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RSP, TMP_ZVAL_OFFSET);
}
prev_opline = opline - 1;
while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
prev_opline--;
}
if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
unknown_num_args = 1;
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_call_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& !JIT_G(current_frame)->call->func) {
call_info = NULL; func = NULL; /* megamorphic call from trait */
}
}
if (!func) {
/* resolve function at run time */
} else if (func->type == ZEND_USER_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
call_num_args = call_info->num_args;
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
call_num_args = call_info->num_args;
} else {
ZEND_UNREACHABLE();
}
if (trace && !func) {
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
#endif
} else if (trace->op == ZEND_JIT_TRACE_ENTER) {
ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
}
}
}
bool may_have_extra_named_params =
opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
(!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
if (!reuse_ip) {
zend_jit_start_reuse_ip();
| // call = EX(call);
| ldr RX, EX->call
}
zend_jit_stop_reuse_ip();
| // fbc = call->func;
| // mov r2, EX:RX->func ???
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, REG0
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| ldr REG0, EX:RX->func
| ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
| TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
| bne &exit_addr
}
}
}
if (!delayed_call_chain) {
if (call_level == 1) {
| str xzr, EX->call
} else {
| //EX(call) = call->prev_execute_data;
| ldr REG0, EX:RX->prev_execute_data
| str REG0, EX->call
}
}
delayed_call_chain = 0;
| //call->prev_execute_data = execute_data;
| str EX, EX:RX->prev_execute_data
if (!func) {
| ldr REG0, EX:RX->func
}
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (!trace) {
| ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
| TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
| bne >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, RX
}
| EXT_CALL zend_jit_deprecated_helper, REG0
| GET_LOW_8BITS RETVALw, RETVALw
| ldr REG0, EX:RX->func // reload
| cbnz RETVALw, >1 // Result is 0 on exception
| b ->exception_handler
|.code
|1:
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, RX
}
| EXT_CALL zend_jit_deprecated_helper, REG0
| cbz RETVALw, ->exception_handler
}
}
if (!func
&& opline->opcode != ZEND_DO_UCALL
&& opline->opcode != ZEND_DO_ICALL) {
| ldrb TMP1w, [REG0, #offsetof(zend_function, type)]
| cmp TMP1w, #ZEND_USER_FUNCTION
| bne >8
}
if ((!func || func->type == ZEND_USER_FUNCTION)
&& opline->opcode != ZEND_DO_ICALL) {
| // EX(call) = NULL;
| str xzr, EX:RX->call
if (RETURN_VALUE_USED(opline)) {
| // EX(return_value) = EX_VAR(opline->result.var);
| LOAD_ZVAL_ADDR REG2, res_addr
| str REG2, EX:RX->return_value
} else {
| // EX(return_value) = 0;
| str xzr, EX:RX->return_value
}
//EX_LOAD_RUN_TIME_CACHE(op_array);
if (!func || func->op_array.cache_size) {
if (func && op_array == &func->op_array) {
/* recursive call */
if (trace || func->op_array.cache_size > sizeof(void*)) {
| ldr REG2, EX->run_time_cache
| str REG2, EX:RX->run_time_cache
}
} else {
if (func
&& !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)
&& ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
| MEM_LOAD_64_ZTS ldr, REG2, compiler_globals, map_ptr_base, TMP1
| ADD_SUB_64_WITH_CONST add, REG2, REG2, (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache), TMP1
| ldr REG2, [REG2]
} else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) ||
(JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) {
/* Closures always use direct pointers */
| ldr REG0, EX:RX->func
| ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)]
} else {
if (func) {
| ldr REG0, EX:RX->func
}
| ldr REG2, [REG0, #offsetof(zend_op_array, run_time_cache__ptr)]
| TST_64_WITH_ONE REG2
| beq >1
| MEM_LOAD_OP_ZTS add, ldr, REG2, compiler_globals, map_ptr_base, REG1, TMP1
| ldr REG2, [REG2]
|1:
}
| str REG2, EX:RX->run_time_cache
}
}
| // EG(current_execute_data) = execute_data;
| MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1
| mov FP, RX
| // opline = op_array->opcodes;
if (func && !unknown_num_args) {
| ADD_SUB_64_WITH_CONST_32 add, TMP1, RX, (EX_NUM_TO_VAR(call_num_args) + offsetof(zval, u1.type_info)), TMP1 // induction variable
for (i = call_num_args; i < func->op_array.last_var; i++) {
| // ZVAL_UNDEF(EX_VAR(n))
| str wzr, [TMP1], #16
}
if (call_num_args <= func->op_array.num_args) {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
uint32_t num_args;
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
if (trace) {
num_args = 0;
} else if (call_info) {
num_args = skip_valid_arguments(op_array, ssa, call_info);
} else {
num_args = call_num_args;
}
} else {
num_args = call_num_args;
}
if (zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes + num_args)
} else {
| ldr REG0, EX->func
|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(num_args * sizeof(zend_op))));
if (GCC_GLOBAL_REGS) {
| ldr IP, [REG0, #offsetof(zend_op_array, opcodes)]
if (num_args) {
| add IP, IP, #(num_args * sizeof(zend_op))
}
} else {
| ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)]
if (num_args) {
| add FCARG1x, FCARG1x, #(num_args * sizeof(zend_op))
}
| str FCARG1x, EX->opline
}
}
if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array
&& num_args >= op_array->required_num_args) {
/* recursive call */
if (ZEND_OBSERVER_ENABLED) {
| SAVE_IP
| mov FCARG1x, FP
| EXT_CALL zend_observer_fcall_begin, REG0
}
#ifdef CONTEXT_THREADED_JIT
| NIY // TODO
#else
| b =>num_args
#endif
return 1;
}
}
} else {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes)
} else if (GCC_GLOBAL_REGS) {
| ldr IP, [REG0, #offsetof(zend_op_array, opcodes)]
} else {
| ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)]
| str FCARG1x, EX->opline
}
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, FP
}
| EXT_CALL zend_jit_copy_extra_args_helper, REG0
}
} else {
| // opline = op_array->opcodes
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes)
} else if (GCC_GLOBAL_REGS) {
| ldr IP, [REG0, #offsetof(zend_op_array, opcodes)]
} else {
| ldr FCARG1x, [REG0, #offsetof(zend_op_array, opcodes)]
| str FCARG1x, EX->opline
}
if (func) {
| // num_args = EX_NUM_ARGS();
| ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)]
| // if (UNEXPECTED(num_args > first_extra_arg))
| CMP_32_WITH_CONST REG1w, (func->op_array.num_args), TMP1w
} else {
| // first_extra_arg = op_array->num_args;
| ldr REG2w, [REG0, #offsetof(zend_op_array, num_args)]
| // num_args = EX_NUM_ARGS();
| ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)]
| // if (UNEXPECTED(num_args > first_extra_arg))
| cmp REG1w, REG2w
}
| bgt >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, FP
}
| EXT_CALL zend_jit_copy_extra_args_helper, REG0
if (!func) {
| ldr REG0, EX->func // reload
}
| ldr REG1w, [FP, #offsetof(zend_execute_data, This.u2.num_args)] // reload
| b >1
|.code
if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
if (!func) {
| // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
| ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
| TST_32_WITH_CONST TMP1w, ZEND_ACC_HAS_TYPE_HINTS, TMP2w
| bne >1
}
| // opline += num_args;
|| ZEND_ASSERT(sizeof(zend_op) == 32);
| mov REG2w, REG1w
| ADD_IP_SHIFT REG2, lsl #5, TMP1
}
|1:
| // if (EXPECTED((int)num_args < op_array->last_var)) {
if (func) {
| LOAD_32BIT_VAL REG2w, func->op_array.last_var
} else {
| ldr REG2w, [REG0, #offsetof(zend_op_array, last_var)]
}
| subs REG2w, REG2w, REG1w
| ble >3
| // zval *var = EX_VAR_NUM(num_args);
| add REG1, FP, REG1, lsl #4
|| ZEND_ASSERT(arm64_may_encode_imm12((int64_t)(ZEND_CALL_FRAME_SLOT * sizeof(zval))));
| add REG1, REG1, #(ZEND_CALL_FRAME_SLOT * sizeof(zval))
|2:
| SET_Z_TYPE_INFO REG1, IS_UNDEF, TMP1w
| add REG1, REG1, #16
| subs REG2w, REG2w, #1
| bne <2
|3:
}
if (ZEND_OBSERVER_ENABLED) {
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
| SET_EX_OPLINE trace[1].opline, REG0
} else {
| SAVE_IP
}
| mov FCARG1x, FP
| EXT_CALL zend_observer_fcall_begin, REG0
}
if (trace) {
if (!func && (opline->opcode != ZEND_DO_UCALL)) {
| b >9
}
} else {
#ifdef CONTEXT_THREADED_JIT
| NIY // TODO: CONTEXT_THREADED_JIT is always undefined.
#else
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP TMP1
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
| JMP_IP TMP1
} else {
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| mov RETVALx, #1 // ZEND_VM_ENTER
| ret
}
}
#endif
}
if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
&& (opline->opcode != ZEND_DO_UCALL)) {
if (!func && (opline->opcode != ZEND_DO_ICALL)) {
|8:
}
if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
| TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
| bne &exit_addr
} else {
| ldr TMP1w, [REG0, #offsetof(zend_op_array, fn_flags)]
| TST_32_WITH_CONST TMP1w, ZEND_ACC_DEPRECATED, TMP2w
| bne >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, RX
}
| EXT_CALL zend_jit_deprecated_helper, REG0
| GET_LOW_8BITS RETVALw, RETVALw
| ldr REG0, EX:RX->func // reload
| cbnz RETVALw, >1 // Result is 0 on exception
| b ->exception_handler
|.code
|1:
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, RX
}
| EXT_CALL zend_jit_deprecated_helper, REG0
| cbz RETVALw, ->exception_handler
| ldr REG0, EX:RX->func // reload
}
}
| // EG(current_execute_data) = execute_data;
| MEM_STORE_64_ZTS str, RX, executor_globals, current_execute_data, REG1
if (ZEND_OBSERVER_ENABLED) {
| mov FCARG1x, RX
| EXT_CALL zend_observer_fcall_begin, REG0
| ldr REG0, EX:RX->func // reload
}
| // ZVAL_NULL(EX_VAR(opline->result.var));
| LOAD_ZVAL_ADDR FCARG2x, res_addr
| SET_Z_TYPE_INFO FCARG2x, IS_NULL, TMP1w
zend_jit_reset_last_valid_opline();
| // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret);
| mov FCARG1x, RX
if (zend_execute_internal) {
| EXT_CALL zend_execute_internal, REG0
} else {
if (func) {
| EXT_CALL func->internal_function.handler, REG0
} else {
| ldr TMP1, [REG0, #offsetof(zend_internal_function, handler)]
| blr TMP1
}
}
if (ZEND_OBSERVER_ENABLED) {
| LOAD_ZVAL_ADDR FCARG2x, res_addr
| mov FCARG1x, RX
| EXT_CALL zend_observer_fcall_end, REG0
}
| // EG(current_execute_data) = execute_data;
| MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0
| // zend_vm_stack_free_args(call);
if (func && !unknown_num_args) {
for (i = 0; i < call_num_args; i++ ) {
if (zend_jit_needs_arg_dtor(func, i, call_info)) {
uint32_t offset = EX_NUM_TO_VAR(i);
zend_jit_addr arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset);
| ZVAL_PTR_DTOR arg_addr, (MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN), 0, 1, opline, ZREG_TMP1, ZREG_TMP2
}
}
} else {
| mov FCARG1x, RX
| EXT_CALL zend_jit_vm_stack_free_args_helper, REG0
}
if (may_have_extra_named_params) {
| ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 3)]
| TST_32_WITH_CONST TMP1w, (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24), TMP2w
| bne >1
|.cold_code
|1:
| ldr FCARG1x, [RX, #offsetof(zend_execute_data, extra_named_params)]
| EXT_CALL zend_free_extra_named_params, REG0
| b >2
|.code
|2:
}
|8:
if (opline->opcode == ZEND_DO_FCALL) {
// TODO: optimize ???
| // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
| ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)]
| TST_32_WITH_CONST TMP1w, (ZEND_CALL_RELEASE_THIS >> 16), TMP2w
| bne >1
|.cold_code
|1:
| add TMP1, RX, #offsetof(zend_execute_data, This)
| GET_Z_PTR FCARG1x, TMP1
| // OBJ_RELEASE(object);
| OBJ_RELEASE ZREG_FCARG1, >2, ZREG_TMP1, ZREG_TMP2
| b >2
|.code
|2:
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!JIT_G(current_frame)->call ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
prev_opline->opcode == ZEND_SEND_UNPACK ||
prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
| // zend_vm_stack_free_call_frame(call);
| ldrb TMP1w, [RX, #(offsetof(zend_execute_data, This.u1.type_info) + 2)]
| TST_32_WITH_CONST TMP1w, ((ZEND_CALL_ALLOCATED >> 16) & 0xff), TMP2w
| bne >1
|.cold_code
|1:
| mov FCARG1x, RX
| EXT_CALL zend_jit_free_call_frame, REG0
| b >1
|.code
}
| MEM_STORE_64_ZTS str, RX, executor_globals, vm_stack_top, REG0
|1:
if (!RETURN_VALUE_USED(opline)) {
zend_class_entry *ce;
bool ce_is_instanceof;
uint32_t func_info = call_info ?
zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
(MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
/* If an exception is thrown, the return_value may stay at the
* original value of null. */
func_info |= MAY_BE_NULL;
if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
| ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline, ZREG_TMP1, ZREG_TMP2
}
}
| // if (UNEXPECTED(EG(exception) != NULL)) {
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbnz REG0, ->icall_throw_handler
// TODO: Can we avoid checking for interrupts after each call ???
if (trace && last_valid_opline != opline) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
exit_addr = NULL;
}
if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) {
return 0;
}
if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
| LOAD_IP_ADDR (opline + 1)
} else if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
| LOAD_IP_ADDR (opline + 1)
}
}
if (!func) {
|9:
}
return 1;
}
static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAL_EX) {
uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
/* Don't generate code that always throws exception */
return 0;
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| ldr REG0, EX:RX->func
| ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
| TST_32_WITH_CONST TMP1w, mask, TMP2w
| bne &exit_addr
} else {
| ldr REG0, EX:RX->func
| ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
| TST_32_WITH_CONST TMP1w, mask, TMP2w
| bne >1
|.cold_code
|1:
if (Z_MODE(op1_addr) == IS_REG) {
/* set type to avoid zval_ptr_dtor() on uninitialized value */
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
| SET_ZVAL_TYPE_INFO addr, IS_UNDEF, TMP1w, TMP2
}
| SET_EX_OPLINE opline, REG0
| b ->throw_cannot_pass_by_ref
|.code
}
}
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, REG0, TMP1
}
} else {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
return 1;
}
static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline)
{
| ldr FCARG1x, EX->call
| ldrb TMP1w, [FCARG1x, #(offsetof(zend_execute_data, This.u1.type_info) + 3)]
| TST_32_WITH_CONST TMP1w, (ZEND_CALL_MAY_HAVE_UNDEF >> 24), TMP2w
| bne >1
|.cold_code
|1:
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_handle_undef_args, REG0
| cbz RETVALw, >2
| b ->exception_handler
|.code
|2:
return 1;
}
static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
{
zend_jit_addr op1_addr, arg_addr, ref_addr;
op1_addr = OP1_ADDR();
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->op1_type == IS_VAR) {
if (op1_info & MAY_BE_INDIRECT) {
| LOAD_ZVAL_ADDR REG0, op1_addr
| // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) {
| IF_NOT_Z_TYPE REG0, IS_INDIRECT, >1, TMP1w
| // ret = Z_INDIRECT_P(ret);
| GET_Z_PTR REG0, REG0
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
}
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2
| b >2
|1:
}
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
}
} else {
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2, ZREG_TMP1
| GET_ZVAL_PTR REG1, op1_addr, TMP1
| GC_ADDREF REG1, TMP1w
| SET_ZVAL_PTR arg_addr, REG1, TMP1
| SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2
| b >6
}
|2:
| // ZVAL_NEW_REF(arg, varptr);
if (opline->op1_type == IS_VAR) {
if (Z_REG(op1_addr) != ZREG_REG0 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR REG0, op1_addr
}
| str REG0, T1 // save
}
| EMALLOC sizeof(zend_reference), op_array, opline // Allocate space in REG0
| mov TMP1w, #2
| str TMP1w, [REG0]
|| ZEND_ASSERT(GC_REFERENCE <= MOVZ_IMM);
| movz TMP1w, #GC_REFERENCE
| str TMP1w, [REG0, #offsetof(zend_reference, gc.u.type_info)]
| str xzr, [REG0, #offsetof(zend_reference, sources.ptr)]
ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val));
if (opline->op1_type == IS_VAR) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0);
| ldr REG1, T1 // restore
| ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| SET_ZVAL_PTR val_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX, TMP1w, TMP2
} else {
| ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| SET_ZVAL_PTR op1_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2
}
| SET_ZVAL_PTR arg_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX, TMP1w, TMP2
}
|6:
| FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline, ZREG_TMP1, ZREG_TMP2
|7:
return 1;
}
static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
arg_num <= MAX_ARG_FLAG_NUM);
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAR_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
| ldr REG0, EX:RX->func
| ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
| TST_32_WITH_CONST TMP1w, mask, TMP2w
| bne >1
|.cold_code
|1:
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
return 0;
}
| b >7
|.code
}
} else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!(op1_info & MAY_BE_REF)) {
/* Don't generate code that always throws exception */
return 0;
} else {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| GET_LOW_8BITS TMP1w, REG1w
| cmp TMP1w, #IS_REFERENCE
| bne &exit_addr
}
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
| ldr REG0, EX:RX->func
| ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
| TST_32_WITH_CONST TMP1w, mask, TMP2w
| bne >1
|.cold_code
|1:
mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
if (op1_info & MAY_BE_REF) {
| GET_LOW_8BITS TMP1w, REG1w
| cmp TMP1w, #IS_REFERENCE
| beq >7
}
| ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
| TST_32_WITH_CONST TMP1w, mask, TMP2w
| bne >7
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| b &exit_addr
} else {
| SET_EX_OPLINE opline, REG0
| LOAD_ZVAL_ADDR FCARG1x, arg_addr
| EXT_CALL zend_jit_only_vars_by_reference, REG0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
| b >7
}
|.code
}
} else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
| ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
| TST_32_WITH_CONST TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
| bne >1
|.cold_code
|1:
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
return 0;
}
| b >7
|.code
}
}
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
|.cold_code
|1:
}
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
| SET_ZVAL_TYPE_INFO arg_addr, IS_NULL, TMP1w, TMP2
| cbz RETVALx, ->exception_handler
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| b >7
|.code
} else {
|7:
return 1;
}
}
if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
if (op1_info & MAY_BE_REF) {
| GET_LOW_8BITS TMP1w, REG1w
| cmp TMP1w, #IS_REFERENCE
| beq >7
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| b &exit_addr
} else {
| SET_EX_OPLINE opline, REG0
| LOAD_ZVAL_ADDR FCARG1x, arg_addr
| EXT_CALL zend_jit_only_vars_by_reference, REG0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF op1_info, REG0w, REG2, TMP1w
} else {
zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8);
| IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1
|.cold_code
|1:
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
| // ZVAL_COPY_VALUE(return_value, &ref->value);
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| GC_DELREF FCARG1x, TMP1w
| beq >1
| IF_NOT_REFCOUNTED REG0w, >2, TMP1w
| GC_ADDREF REG2, TMP1w
| b >2
|1:
| EFREE_REFERENCE
| b >2
|.code
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
|2:
}
} else {
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr= op1_def_addr;
}
}
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
if (opline->op1_type == IS_CV) {
| TRY_ADDREF op1_info, REG0w, REG2, TMP1w
}
}
}
|7:
return 1;
}
static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline)
{
uint32_t arg_num = opline->op2.num;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
| // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
|| if (reuse_ip) {
| ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
| BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
| str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
|| } else {
| ldr REG0, EX->call
| ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
| BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
| str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
|| }
}
} else {
if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
| // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
|| if (reuse_ip) {
| ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
| BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w
| str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
|| } else {
| ldr REG0, EX->call
| ldr TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
| BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~ZEND_CALL_SEND_ARG_BY_REF), TMP2w
| str TMP1w, [REG0, #offsetof(zend_execute_data, This.u1.type_info)]
|| }
}
}
} else {
// if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
| ldr REG0, EX:RX->func
| ldr TMP1w, [REG0, #offsetof(zend_function, quick_arg_flags)]
| TST_32_WITH_CONST TMP1w, mask, TMP2w
| bne >1
|.cold_code
|1:
| // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
| ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
| BW_OP_32_WITH_CONST orr, TMP1w, TMP1w, ZEND_CALL_SEND_ARG_BY_REF, TMP2w
| str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
| b >1
|.code
| // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
| ldr TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
| BW_OP_32_WITH_CONST and, TMP1w, TMP1w, (~(ZEND_CALL_SEND_ARG_BY_REF)), TMP2w
| str TMP1w, [RX, #offsetof(zend_execute_data, This.u1.type_info)]
|1:
}
return 1;
}
static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
if (jmp) {
| b >7
}
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| b =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
if (jmp) {
| b >7
}
}
return 1;
}
static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
if (jmp) {
| b >7
}
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
if (jmp) {
| b >7
}
}
return 1;
}
static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t defined_label = (uint32_t)-1;
uint32_t undefined_label = (uint32_t)-1;
zval *zv = RT_CONSTANT(opline, opline->op1);
zend_jit_addr res_addr = 0;
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
undefined_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
defined_label = target_label;
} else {
ZEND_UNREACHABLE();
}
}
| // if (CACHED_PTR(opline->extended_value)) {
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, opline->extended_value, TMP1
| cbz REG0, >1
| TST_64_WITH_ONE REG0
| bne >4
|.cold_code
|4:
| MEM_LOAD_64_ZTS ldr, FCARG1x, executor_globals, zend_constants, FCARG1x
| ldr TMP1w, [FCARG1x, #offsetof(HashTable, nNumOfElements)]
| cmp TMP1, REG0, lsr #1
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| beq &exit_addr
} else {
| beq >3
}
} else if (undefined_label != (uint32_t)-1) {
| beq =>undefined_label
} else {
| beq >3
}
} else {
| beq >2
}
|1:
| SET_EX_OPLINE opline, REG0
| LOAD_ADDR FCARG1x, zv
| EXT_CALL zend_jit_check_constant, REG0
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| cbz RETVALx, >3
} else {
| cbnz RETVALx, >3
}
| b &exit_addr
} else if (smart_branch_opcode) {
if (undefined_label != (uint32_t)-1) {
| cbz RETVALx, =>undefined_label
} else {
| cbz RETVALx, >3
}
if (defined_label != (uint32_t)-1) {
| b =>defined_label
} else {
| b >3
}
} else {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| cbnz RETVALx, >1
|2:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
| b >3
}
|.code
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
}
} else if (defined_label != (uint32_t)-1) {
| b =>defined_label
}
} else {
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
}
|3:
return 1;
}
static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t mask;
zend_jit_addr op1_addr = OP1_ADDR();
// TODO: support for is_resource() ???
ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
|.cold_code
|1:
}
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
zend_jit_check_exception_undef_result(Dst, opline);
if (opline->extended_value & MAY_BE_NULL) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
| b >7
}
} else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
} else {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b &exit_addr
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
| b >7
}
} else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) {
return 0;
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
|.code
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
mask = opline->extended_value;
if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
}
} else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
} else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b &exit_addr
}
} else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) {
return 0;
}
} else {
bool invert = 0;
uint8_t type;
switch (mask) {
case MAY_BE_NULL: type = IS_NULL; break;
case MAY_BE_FALSE: type = IS_FALSE; break;
case MAY_BE_TRUE: type = IS_TRUE; break;
case MAY_BE_LONG: type = IS_LONG; break;
case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
case MAY_BE_STRING: type = IS_STRING; break;
case MAY_BE_ARRAY: type = IS_ARRAY; break;
case MAY_BE_OBJECT: type = IS_OBJECT; break;
case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break;
case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break;
case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break;
case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break;
case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break;
case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
default:
type = 0;
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR REG0, op1_addr
| ZVAL_DEREF REG0, op1_info, TMP1w
}
if (type == 0) {
if (smart_branch_opcode &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
| IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2
|.cold_code
|1:
}
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
| GC_DELREF FCARG1x, TMP1w
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
| bne >3
}
if (op1_info & MAY_BE_REF) {
| ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)]
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
}
| str REG0w, T1 // save
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op1_info, opline, TMP1
| ldr REG1w, T1 // restore
| b >2
}
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (!RC_MAY_BE_1(op1_info)) {
| b >3
}
|.code
}
|3:
if (op1_info & MAY_BE_REF) {
| ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)]
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
}
|2:
} else {
if (op1_info & MAY_BE_REF) {
| ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)]
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
}
}
| mov REG0w, #1
| lsl REG0w, REG0w, REG1w
| TST_32_WITH_CONST REG0w, mask, TMP1w
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| bne &exit_addr
} else {
| beq &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| beq =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| bne =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| cset REG0w, ne
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
} else {
if (smart_branch_opcode &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
| IF_ZVAL_REFCOUNTED op1_addr, >1, ZREG_TMP1, ZREG_TMP2
|.cold_code
|1:
}
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
| GC_DELREF FCARG1x, TMP1w
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
| bne >3
}
if (op1_info & MAY_BE_REF) {
| ldrb REG0w, [REG0, #offsetof(zval,u1.v.type)]
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, REG0w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
}
| str REG0w, T1 // save
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op1_info, opline, TMP1
| ldr REG1w, T1 // restore
| b >2
}
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (!RC_MAY_BE_1(op1_info)) {
| b >3
}
|.code
}
|3:
if (op1_info & MAY_BE_REF) {
| ldrb REG1w, [REG0, #offsetof(zval,u1.v.type)]
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
}
|2:
// Note: 'type' is of uchar type and holds a positive value,
// hence it's safe to directly encode it as the imm field of 'cmp' instruction.
| cmp REG1w, #type
} else {
if (op1_info & MAY_BE_REF) {
| ldrb TMP1w, [REG0, #offsetof(zval,u1.v.type)]
| cmp TMP1w, #type
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, FP, (opline->op1.var + offsetof(zval,u1.v.type)), TMP1
| cmp TMP1w, #type
}
}
if (exit_addr) {
if (invert) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| bne &exit_addr
} else {
| beq &exit_addr
}
} else {
if (smart_branch_opcode == ZEND_JMPNZ) {
| beq &exit_addr
} else {
| bne &exit_addr
}
}
} else if (smart_branch_opcode) {
if (invert) {
if (smart_branch_opcode == ZEND_JMPZ) {
| beq =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| bne =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
if (smart_branch_opcode == ZEND_JMPZ) {
| bne =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| beq =>target_label
} else {
ZEND_UNREACHABLE();
}
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (invert) {
| cset REG0w, ne
} else {
| cset REG0w, eq
}
| add REG0w, REG0w, #2
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
}
}
}
|7:
return 1;
}
static int zend_jit_leave_frame(dasm_State **Dst)
{
| // EG(current_execute_data) = EX(prev_execute_data);
| ldr REG0, EX->prev_execute_data
| MEM_STORE_64_ZTS str, REG0, executor_globals, current_execute_data, REG2
return 1;
}
static int zend_jit_free_cvs(dasm_State **Dst)
{
| // EG(current_execute_data) = EX(prev_execute_data);
| ldr FCARG1x, EX->prev_execute_data
| MEM_STORE_64_ZTS str, FCARG1x, executor_globals, current_execute_data, REG0
| // zend_free_compiled_variables(execute_data);
| mov FCARG1x, FP
| EXT_CALL zend_free_compiled_variables, REG0
return 1;
}
static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
uint32_t offset = EX_NUM_TO_VAR(var);
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset);
| ZVAL_PTR_DTOR addr, info, 1, 1, NULL, ZREG_TMP1, ZREG_TMP2
}
return 1;
}
static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset);
| ZVAL_PTR_DTOR addr, info, 0, 1, opline, ZREG_TMP1, ZREG_TMP2
}
return 1;
}
static int zend_jit_leave_func(dasm_State **Dst,
const zend_op_array *op_array,
const zend_op *opline,
uint32_t op1_info,
bool left_frame,
zend_jit_trace_rec *trace,
zend_jit_trace_info *trace_info,
int indirect_var_access,
int may_throw)
{
bool may_be_top_frame =
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
bool may_need_call_helper =
indirect_var_access || /* may have symbol table */
!op_array->function_name || /* may have symbol table */
may_be_top_frame ||
(op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
(uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
bool may_need_release_this =
!(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
op_array->scope &&
!(op_array->fn_flags & ZEND_ACC_STATIC) &&
(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
if (may_need_call_helper || may_need_release_this) {
| ldr FCARG1w, [FP, #offsetof(zend_execute_data, This.u1.type_info)]
}
if (may_need_call_helper) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
/* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
| TST_32_WITH_CONST FCARG1w, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE), TMP1w
if (trace && trace->op != ZEND_JIT_TRACE_END) {
| bne >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1x, FP
}
| EXT_CALL zend_jit_leave_func_helper, REG0
if (may_be_top_frame) {
// TODO: try to avoid this check ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
/* this check should be handled by the following OPLINE guard */
| LOAD_ADDR TMP1, zend_jit_halt_op
| cmp IP, TMP1
| beq ->trace_halt
#endif
} else if (GCC_GLOBAL_REGS) {
| cbz IP, ->trace_halt
} else {
| tst RETVALw, RETVALw
| blt ->trace_halt
}
}
if (!GCC_GLOBAL_REGS) {
| // execute_data = EG(current_execute_data)
| MEM_LOAD_64_ZTS ldr, FP, executor_globals, current_execute_data, TMP1
}
| b >8
|.code
} else {
| bne ->leave_function_handler
}
}
if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
| // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
| ldr FCARG1x, EX->func
| sub FCARG1x, FCARG1x, #sizeof(zend_object)
| OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2
|4:
} else if (may_need_release_this) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) {
| // if (call_info & ZEND_CALL_RELEASE_THIS)
| TST_32_WITH_CONST FCARG1w, ZEND_CALL_RELEASE_THIS, TMP1w
| beq >4
}
| // zend_object *object = Z_OBJ(execute_data->This);
| ldr FCARG1x, EX->This.value.obj
| // OBJ_RELEASE(object);
| OBJ_RELEASE ZREG_FCARG1, >4, ZREG_TMP1, ZREG_TMP2
|4:
// TODO: avoid EG(excption) check for $this->foo() calls
may_throw = 1;
}
| // EG(vm_stack_top) = (zval*)execute_data;
| MEM_STORE_64_ZTS str, FP, executor_globals, vm_stack_top, REG0
| // execute_data = EX(prev_execute_data);
| ldr FP, EX->prev_execute_data
if (!left_frame) {
| // EG(current_execute_data) = execute_data;
| MEM_STORE_64_ZTS str, FP, executor_globals, current_execute_data, REG0
}
|9:
if (trace) {
if (trace->op != ZEND_JIT_TRACE_END
&& (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
zend_jit_reset_last_valid_opline();
} else {
| LOAD_IP
| ADD_IP_WITH_CONST sizeof(zend_op), TMP1
}
|8:
if (trace->op == ZEND_JIT_TRACE_BACK
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
const zend_op *next_opline = trace->opline;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
/* exception might be thrown during destruction of unused return value */
| // if (EG(exception))
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbnz REG0, ->leave_throw_handler
}
do {
trace++;
} while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
next_opline = trace->opline;
ZEND_ASSERT(next_opline != NULL);
if (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
trace_info->flags |= ZEND_JIT_TRACE_LOOP;
| CMP_IP next_opline, TMP1, TMP2
| beq =>0 // LOOP
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| JMP_IP TMP1
#else
| b ->trace_escape
#endif
} else {
| CMP_IP next_opline, TMP1, TMP2
| bne ->trace_escape
}
zend_jit_set_last_valid_opline(trace->opline);
return 1;
} else if (may_throw ||
(((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
| // if (EG(exception))
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| cbnz REG0, ->leave_throw_handler
}
return 1;
} else {
| // if (EG(exception))
| MEM_LOAD_64_ZTS ldr, REG0, executor_globals, exception, TMP1
| LOAD_IP
| cbnz REG0, ->leave_throw_handler
| // opline = EX(opline) + 1
| ADD_IP_WITH_CONST sizeof(zend_op), TMP1
}
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
#ifdef CONTEXT_THREADED_JIT
| NIY // TODO: CONTEXT_THREADED_JIT is always undefined
#else
| JMP_IP TMP1
#endif
} else if (GCC_GLOBAL_REGS) {
| ldp x29, x30, [sp], # SPAD // stack alignment
#ifdef CONTEXT_THREADED_JIT
| NIY // TODO
#else
| JMP_IP TMP1
#endif
} else {
#ifdef CONTEXT_THREADED_JIT
ZEND_UNREACHABLE();
// TODO: context threading can't work without GLOBAL REGS because we have to change
// the value of execute_data in execute_ex()
| NIY // TODO
#else
| ldp FP, RX, T2 // restore FP and IP
| ldp x29, x30, [sp], # NR_SPAD // stack alignment
| mov RETVALx, #2 // ZEND_VM_LEAVE ????
| ret
#endif
}
return 1;
}
static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
{
zend_jit_addr ret_addr;
int8_t return_value_used;
ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
return_value_used = 1;
} else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
return_value_used = 0;
} else {
return_value_used = -1;
}
} else {
return_value_used = -1;
}
if (ZEND_OBSERVER_ENABLED) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) {
return 0;
}
op1_addr = dst;
}
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
| mov FCARG1x, FP
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_observer_fcall_end, REG0
}
// if (!EX(return_value))
if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_REG1) {
if (return_value_used != 0) {
| ldr REG2, EX->return_value
}
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG2, 0);
} else {
if (return_value_used != 0) {
| ldr REG1, EX->return_value
}
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG1, 0);
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (return_value_used == -1) {
| cbz Rx(Z_REG(ret_addr)), >1
|.cold_code
|1:
}
if (return_value_used != 1) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (jit_return_label >= 0) {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label, ZREG_TMP1, ZREG_TMP2
} else {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, >9, ZREG_TMP1, ZREG_TMP2
}
}
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
| GC_DELREF FCARG1x, TMP1w
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
if (jit_return_label >= 0) {
| bne =>jit_return_label
} else {
| bne >9
}
}
| //SAVE_OPLINE()
| ZVAL_DTOR_FUNC op1_info, opline, TMP1
| //????ldr REG1, EX->return_value // reload ???
}
if (return_value_used == -1) {
if (jit_return_label >= 0) {
| b =>jit_return_label
} else {
| b >9
}
|.code
}
}
} else if (return_value_used == -1) {
if (jit_return_label >= 0) {
| cbz Rx(Z_REG(ret_addr)), =>jit_return_label
} else {
| cbz Rx(Z_REG(ret_addr)), >9
}
}
if (return_value_used == 0) {
|9:
return 1;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, REG0, TMP1
}
} else if (opline->op1_type == IS_TMP_VAR) {
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR REG0, op1_addr
| ZVAL_DEREF REG0, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
}
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
!op_array->function_name) {
| TRY_ADDREF op1_info, REG0w, REG2, TMP1w
} else if (return_value_used != 1) {
| // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL, TMP1w, TMP2
}
}
} else {
if (op1_info & MAY_BE_REF) {
zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, offsetof(zend_reference, val));
| IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1, ZREG_TMP1
|.cold_code
|1:
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR REG0, op1_addr, TMP1
| // ZVAL_COPY_VALUE(return_value, &ref->value);
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_REG2, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| GC_DELREF REG0, TMP1w
| beq >2
| // if (IS_REFCOUNTED())
if (jit_return_label >= 0) {
| IF_NOT_REFCOUNTED REG2w, =>jit_return_label, TMP1w
} else {
| IF_NOT_REFCOUNTED REG2w, >9, TMP1w
}
| // ADDREF
| GET_ZVAL_PTR REG2, ret_addr, TMP1 // reload
| GC_ADDREF REG2, TMP1w
if (jit_return_label >= 0) {
| b =>jit_return_label
} else {
| b >9
}
|2:
| mov FCARG1x, REG0
| EFREE_REFERENCE
if (jit_return_label >= 0) {
| b =>jit_return_label
} else {
| b >9
}
|.code
}
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
|9:
return 1;
}
static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg)
{
ZEND_ASSERT(type_reg == ZREG_REG2);
| GET_ZVAL_PTR REG1, val_addr, TMP1
| IF_NOT_REFCOUNTED REG2w, >2, TMP1w
| GET_LOW_8BITS TMP2w, REG2w
| IF_NOT_TYPE TMP2w, IS_REFERENCE, >1
| add REG1, REG1, #offsetof(zend_reference, val)
| GET_Z_TYPE_INFO REG2w, REG1
| GET_Z_PTR REG1, REG1
| IF_NOT_REFCOUNTED REG2w, >2, TMP1w
|1:
| GC_ADDREF REG1, TMP2w
|2:
| SET_ZVAL_PTR res_addr, REG1, TMP1
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1
return 1;
}
static int zend_jit_fetch_dim_read(dasm_State **Dst,
const zend_op *opline,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
uint32_t res_info,
zend_jit_addr res_addr,
uint8_t dim_type)
{
zend_jit_addr orig_op1_addr, op2_addr;
const void *exit_addr = NULL;
const void *not_found_exit_addr = NULL;
const void *res_exit_addr = NULL;
bool result_avoid_refcounting = 0;
uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
int may_throw = 0;
orig_op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
if (opline->opcode != ZEND_FETCH_DIM_IS
&& JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if ((res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
uint32_t flags = 0;
uint32_t old_op1_info = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
if (opline->opcode != ZEND_FETCH_LIST_R
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !op1_avoid_refcounting) {
flags |= ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
flags |= ZEND_JIT_EXIT_FREE_OP2;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
if (op1_avoid_refcounting) {
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!res_exit_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
}
if (opline->opcode == ZEND_FETCH_DIM_IS
&& !(res_info & MAY_BE_NULL)) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
}
if (op1_avoid_refcounting) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
}
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
}
}
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
(opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) {
return 0;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1
}
}
| SET_EX_OPLINE opline, REG0
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
if (opline->opcode != ZEND_FETCH_DIM_IS) {
if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr, TMP1
| EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, REG0
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
| EXT_CALL zend_jit_fetch_dim_str_r_helper, REG0
}
| SET_ZVAL_PTR res_addr, RETVALx, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
| LOAD_ZVAL_ADDR CARG3, res_addr
| EXT_CALL zend_jit_fetch_dim_str_is_helper, REG0
}
if ((op1_info & MAY_BE_ARRAY) ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) {
| b >9 // END
}
|6:
}
if (op1_info & MAY_BE_OBJECT) {
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
if (exit_addr) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6, ZREG_TMP1
}
}
| SET_EX_OPLINE opline, REG0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
| LOAD_ZVAL_ADDR CARG3, res_addr
if (opline->opcode != ZEND_FETCH_DIM_IS) {
| EXT_CALL zend_jit_fetch_dim_obj_r_helper, REG0
} else {
| EXT_CALL zend_jit_fetch_dim_obj_is_helper, REG0
}
if ((op1_info & MAY_BE_ARRAY) ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
| b >9 // END
}
|6:
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
&& (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
| SET_EX_OPLINE opline, REG0
if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
may_throw = 1;
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
|1:
}
if (op2_info & MAY_BE_UNDEF) {
may_throw = 1;
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1
| LOAD_32BIT_VAL FCARG1w, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
|1:
}
}
if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
may_throw = 1;
if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
| LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr
} else {
| SET_EX_OPLINE opline, REG0
if (Z_MODE(op1_addr) != IS_MEM_ZVAL ||
Z_REG(op1_addr) != ZREG_FCARG1 ||
Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
}
| EXT_CALL zend_jit_invalid_array_access, REG0
}
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
if (op1_info & MAY_BE_ARRAY) {
| b >9 // END
}
}
if (op1_info & MAY_BE_ARRAY) {
|.code
}
}
if (op1_info & MAY_BE_ARRAY) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
|8:
if (res_exit_addr) {
uint32_t type = concrete_type(res_info);
if ((op1_info & MAY_BE_ARRAY_OF_REF)
&& dim_type != IS_UNKNOWN
&& dim_type != IS_REFERENCE) {
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1
|.cold_code
|1:
| IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr, ZREG_TMP1
| GET_Z_PTR REG0, REG0
| add REG0, REG0, #offsetof(zend_reference, val)
| IF_ZVAL_TYPE val_addr, type, >1, ZREG_TMP1
| b &res_exit_addr
|.code
|1:
} else {
| GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
| GET_LOW_8BITS TMP1w, REG2w
| IF_NOT_TYPE TMP1w, type, >1
|.cold_code
|1:
| IF_NOT_TYPE TMP1w, IS_REFERENCE, &res_exit_addr
| GET_Z_PTR REG0, REG0
| add REG0, REG0, #offsetof(zend_reference, val)
| GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
| GET_LOW_8BITS TMP1w, REG2w
| IF_TYPE TMP1w, type, >1
| b &res_exit_addr
|.code
|1:
}
} else {
if (op1_info & MAY_BE_ARRAY_OF_REF) {
| ZVAL_DEREF REG0, MAY_BE_REF, TMP1w
}
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr, ZREG_TMP1
} else {
| GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
| GET_LOW_8BITS TMP1w, REG2w
| IF_NOT_TYPE TMP1w, type, &res_exit_addr
}
}
| // ZVAL_COPY
|7:
| ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (type < IS_STRING) {
if (Z_REG(res_addr) != ZREG_FP ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
| SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2
}
} else {
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1
if (!result_avoid_refcounting) {
| TRY_ADDREF res_info, REG2w, REG1, TMP1w
}
}
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
} else if (op1_info & MAY_BE_ARRAY_OF_REF) {
| // ZVAL_COPY_DEREF
| GET_ZVAL_TYPE_INFO Rw(ZREG_REG2), val_addr, TMP1
if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_REG2)) {
return 0;
}
} else {
| // ZVAL_COPY
| ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF res_info, REG1w, REG2, TMP1w
}
}
|9: // END
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetGet() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
}
if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
}
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_fetch_dim(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
uint8_t dim_type)
{
zend_jit_addr op2_addr;
int may_throw = 0;
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
if (opline->opcode == ZEND_FETCH_DIM_RW) {
| SET_EX_OPLINE opline, REG0
}
if (op1_info & MAY_BE_REF) {
may_throw = 1;
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE FCARG1x, IS_REFERENCE, >1, TMP1w
| GET_Z_PTR FCARG2x, FCARG1x
| ldrb TMP1w, [FCARG2x, #(offsetof(zend_reference, val) + offsetof(zval, u1.v.type))]
| cmp TMP1w, #IS_ARRAY
| bne >2
| add FCARG1x, FCARG2x, #offsetof(zend_reference, val)
| b >3
|.cold_code
|2:
| SET_EX_OPLINE opline, REG0
if (opline->opcode != ZEND_FETCH_DIM_RW) {
| EXT_CALL zend_jit_prepare_assign_dim_ref, REG0
}
| mov FCARG1x, RETVALx
| cbnz FCARG1x, >1
| b ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1, ZREG_TMP1, ZREG_TMP2
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL, ZREG_TMP1
| bgt >7
}
if (Z_REG(op1_addr) != ZREG_FP) {
| str Rx(Z_REG(op1_addr)), T1 // save
}
if ((op1_info & MAY_BE_UNDEF)
&& opline->opcode == ZEND_FETCH_DIM_RW) {
may_throw = 1;
if (op1_info & MAY_BE_NULL) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
}
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
|1:
}
| // ZVAL_ARR(container, zend_new_array(8));
| EXT_CALL _zend_new_array_0, REG0
| mov REG0, RETVALx
if (Z_REG(op1_addr) != ZREG_FP) {
| ldr Rx(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL_FROM_REG op1_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX, TMP1w, TMP2
| mov FCARG1x, REG0
if (op1_info & MAY_BE_ARRAY) {
| b >1
|.code
|1:
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|6:
if (opline->op2_type == IS_UNUSED) {
may_throw = 1;
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2x, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, REG0
| // if (UNEXPECTED(!var_ptr)) {
| cbz RETVALx, >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| b >8
|.code
| SET_ZVAL_PTR res_addr, RETVALx, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2
} else {
uint32_t type;
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
type = BP_VAR_W;
break;
case ZEND_FETCH_DIM_RW:
may_throw = 1;
type = BP_VAR_RW;
break;
case ZEND_FETCH_DIM_UNSET:
type = BP_VAR_UNSET;
break;
default:
ZEND_UNREACHABLE();
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) {
return 0;
}
|8:
| SET_ZVAL_PTR res_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2
if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
|.cold_code
|9:
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
| b >8
|.code
}
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
may_throw = 1;
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
if (opline->opcode != ZEND_FETCH_DIM_RW) {
| SET_EX_OPLINE opline, REG0
}
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| mov FCARG2x, xzr
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
| LOAD_ZVAL_ADDR CARG3, res_addr
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
| EXT_CALL zend_jit_fetch_dim_obj_w_helper, REG0
break;
case ZEND_FETCH_DIM_RW:
| EXT_CALL zend_jit_fetch_dim_obj_rw_helper, REG0
break;
// case ZEND_FETCH_DIM_UNSET:
// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, REG0
// break;
default:
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
| b >8 // END
|.code
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & MAY_HAVE_DTOR)
&& (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
|8:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_isset_isempty_dim(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
uint8_t dim_type,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr)
{
zend_jit_addr op2_addr, res_addr;
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
op2_addr = OP2_ADDR();
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
const void *found_exit_addr = NULL;
const void *not_found_exit_addr = NULL;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
}
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1
if (exit_addr
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
&& !may_throw
&& (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
&& (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
if (smart_branch_opcode == ZEND_JMPNZ) {
found_exit_addr = exit_addr;
} else {
not_found_exit_addr = exit_addr;
}
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) {
return 0;
}
if (found_exit_addr) {
|9:
return 1;
} else if (not_found_exit_addr) {
|8:
return 1;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
| SET_EX_OPLINE opline, REG0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2x, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2x, op2_addr
}
| EXT_CALL zend_jit_isset_dim_helper, REG0
| cbz RETVALw, >9
if (op1_info & MAY_BE_ARRAY) {
| b >8
|.code
}
} else {
if (op2_info & MAY_BE_UNDEF) {
if (op2_info & MAY_BE_ANY) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1, ZREG_TMP1
}
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
|1:
}
if (op1_info & MAY_BE_ARRAY) {
| b >9
|.code
}
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetExists() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) {
|8:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
if (!op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
}
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| b &exit_addr
} else {
| b >8
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b =>target_label2
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| b =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
| b >8
}
} else {
| NIY // TODO: support for empty()
}
}
|9: // not found
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline, ZREG_TMP1, ZREG_TMP2
if (!op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
}
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| b =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
} else {
ZEND_UNREACHABLE();
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
}
} else {
| NIY // TODO: support for empty()
}
|8:
return 1;
}
static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr op1_addr = OP1_ADDR();
zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
| // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1;
| ldr FCARG2x, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG2x, opline->extended_value, TMP1
| sub REG0, REG0, #1
| // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
| MEM_LOAD_32_ZTS ldr, REG1w, executor_globals, symbol_table.nNumUsed, REG1
| cmp REG0, REG1, lsl #5
| bhs >9
| // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
| MEM_LOAD_64_ZTS ldr, TMP1, executor_globals, symbol_table.arData, REG1
| add REG0, REG0, TMP1
| IF_NOT_Z_TYPE REG0, IS_REFERENCE, >9, TMP1w
| // (EXPECTED(p->key == varname))
| ldr TMP1, [REG0, #offsetof(Bucket, key)]
| LOAD_ADDR TMP2, varname
| cmp TMP1, TMP2
| bne >9
| GET_Z_PTR REG0, REG0
| GC_ADDREF REG0, TMP1w
|1:
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
| IF_ZVAL_REFCOUNTED op1_addr, >2, ZREG_TMP1, ZREG_TMP2
|.cold_code
|2:
}
| // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
| // ZVAL_REF(variable_ptr, ref)
| SET_ZVAL_PTR op1_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2
| // if (GC_DELREF(garbage) == 0)
| GC_DELREF FCARG1x, TMP1w
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
| bne >3
} else {
| bne >5
}
| ZVAL_DTOR_FUNC op1_info, opline, TMP1
| b >5
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
|3:
| // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
| IF_GC_MAY_NOT_LEAK FCARG1x, >5, TMP1w, TMP2w
| EXT_CALL gc_possible_root, REG0
| b >5
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|.code
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // ZVAL_REF(variable_ptr, ref)
| SET_ZVAL_PTR op1_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX, TMP1w, TMP2
}
|5:
//END of handler
|.cold_code
|9:
| LOAD_ADDR FCARG1x, (ptrdiff_t)varname
if (opline->extended_value) {
| ADD_SUB_64_WITH_CONST_32 add, FCARG2x, FCARG2x, opline->extended_value, TMP1
}
| EXT_CALL zend_jit_fetch_global_helper, REG0
| mov REG0, RETVALx
| b <1
|.code
return 1;
}
static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
bool in_cold = 0;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_REG0;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->prev) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
return 1;
}
}
if (ZEND_ARG_SEND_MODE(arg_info)) {
if (opline->opcode == ZEND_RECV_INIT) {
| LOAD_ZVAL_ADDR Rx(tmp_reg), res_addr
| ZVAL_DEREF Rx(tmp_reg), MAY_BE_REF, TMP1w
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0);
} else {
| GET_ZVAL_PTR Rx(tmp_reg), res_addr, TMP1
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val));
}
}
if (type_mask != 0) {
if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
| IF_NOT_ZVAL_TYPE res_addr, type_code, >1, ZREG_TMP1
} else {
| mov REG2w, #1
| MEM_ACCESS_8_WITH_UOFFSET ldrb, REG1w, Rx(Z_REG(res_addr)), Z_OFFSET(res_addr)+offsetof(zval, u1.v.type), TMP1
| lsl REG2w, REG2w, REG1w
| TST_32_WITH_CONST REG2w, type_mask, TMP1w
| beq >1
}
|.cold_code
|1:
in_cold = 1;
}
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, res_addr
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, REG0
} else {
| ADDR_STORE EX->opline, opline, REG0
}
| LOAD_ADDR FCARG2x, (ptrdiff_t)arg_info
| EXT_CALL zend_jit_verify_arg_slow, REG0
if (check_exception) {
| GET_LOW_8BITS REG0w, RETVALw
if (in_cold) {
| cbnz REG0w, >1
| b ->exception_handler
|.code
|1:
} else {
| cbz REG0w, ->exception_handler
}
} else if (in_cold) {
| b >1
|.code
|1:
}
return 1;
}
static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array)
{
uint32_t arg_num = opline->op1.num;
zend_arg_info *arg_info = NULL;
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
if (EXPECTED(arg_num <= op_array->num_args)) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
arg_info = &op_array->arg_info[op_array->num_args];
}
if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
arg_info = NULL;
}
}
if (arg_info || (opline+1)->opcode != ZEND_RECV) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| ldr TMP1w, EX->This.u2.num_args
| CMP_32_WITH_CONST TMP1w, arg_num, TMP2w
| blo &exit_addr
}
} else {
| ldr TMP1w, EX->This.u2.num_args
| CMP_32_WITH_CONST TMP1w, arg_num, TMP2w
| blo >1
|.cold_code
|1:
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, REG0
} else {
| ADDR_STORE EX->opline, opline, REG0
}
| mov FCARG1x, FP
| EXT_CALL zend_missing_arg_error, REG0
| b ->exception_handler
|.code
}
}
if (arg_info) {
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) {
return 0;
}
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
| LOAD_IP_ADDR (opline + 1)
zend_jit_set_last_valid_opline(opline + 1);
}
}
return 1;
}
static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
{
uint32_t arg_num = opline->op1.num;
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
| ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, REG0, TMP1
}
}
} else {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| ldr TMP1w, EX->This.u2.num_args
| CMP_32_WITH_CONST TMP1w, arg_num, TMP2w
| bhs >5
}
| ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, REG0, TMP1
}
}
if (Z_CONSTANT_P(zv)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, REG0
} else {
| ADDR_STORE EX->opline, opline, REG0
}
| LOAD_ZVAL_ADDR FCARG1x, res_addr
| ldr REG0, EX->func
| ldr FCARG2x, [REG0, #offsetof(zend_op_array, scope)]
| EXT_CALL zval_update_constant_ex, REG0
| cbnz RETVALw, >1
|.cold_code
|1:
| ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline, ZREG_TMP1, ZREG_TMP2
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF, TMP1w, TMP2
| b ->exception_handler
|.code
}
|5:
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
do {
zend_arg_info *arg_info;
if (arg_num <= op_array->num_args) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
arg_info = &op_array->arg_info[op_array->num_args];
} else {
break;
}
if (!ZEND_TYPE_IS_SET(arg_info->type)) {
break;
}
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) {
return 0;
}
} while (0);
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (is_last) {
| LOAD_IP_ADDR (opline + 1)
zend_jit_set_last_valid_opline(opline + 1);
}
}
return 1;
}
static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| LOAD_ADDR TMP1, ((ptrdiff_t)ce)
| ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)]
| cmp TMP2, TMP1
| bne &exit_addr
return 1;
}
static int zend_jit_fetch_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
bool op1_avoid_refcounting,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_property_info *prop_info;
bool may_be_dynamic = 1;
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
uint32_t res_info = RES_INFO();
bool type_loaded = 0;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1x, this_addr, TMP1
} else {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
| GET_Z_PTR FCARG1x, FCARG1x
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7, ZREG_TMP1
}
}
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
| cmp REG2, TMP1
| bne >5
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)), TMP1
may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array);
if (may_be_dynamic) {
| tst REG0, REG0
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| blt >5
} else {
| blt >8 // dynamic property
}
}
| add TMP1, FCARG1x, REG0
| ldr REG2w, [TMP1, #offsetof(zval,u1.type_info)]
| IF_UNDEF REG2w, >5
| mov FCARG1x, TMP1
type_loaded = 1;
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, ((opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2), TMP1
| cbnz FCARG2x, >1
|.cold_code
|1:
| ldr TMP1w, [FCARG2x, #offsetof(zend_property_info, flags)]
| tst TMP1w, #ZEND_ACC_READONLY
if (flags) {
| beq >3
} else {
| beq >4
}
| IF_NOT_TYPE REG2w, IS_OBJECT_EX, >2
| GET_Z_PTR REG2, FCARG1x
| GC_ADDREF REG2, TMP1w
| SET_ZVAL_PTR res_addr, REG2, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2
| b >9
|2:
| ldr REG0w, [FCARG1x, #offsetof(zval, u2.extra)]
| TST_32_WITH_CONST REG0w, IS_PROP_REINITABLE, TMP1w
| beq >6
| and REG0w, REG0w, #(~IS_PROP_REINITABLE)
| str REG0w, [FCARG1x, #offsetof(zval, u2.extra)]
if (flags) {
| b >3
} else {
| b >4
}
|6:
| mov FCARG1x, FCARG2x
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_readonly_property_modification_error, REG0
| SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2
| b >9
|3:
if (flags == ZEND_FETCH_DIM_WRITE) {
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_jit_check_array_promotion, REG0
| b >9
} else if (flags == ZEND_FETCH_REF) {
| LOAD_ZVAL_ADDR CARG3, res_addr
| EXT_CALL zend_jit_create_typed_ref, REG0
| b >9
} else {
ZEND_ASSERT(flags == 0);
}
|.code
|4:
}
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
type_loaded = 1;
| MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
| IF_UNDEF REG2w, &exit_addr
}
} else {
type_loaded = 1;
| MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
| IF_UNDEF REG2w, >5
}
if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
if (!type_loaded) {
type_loaded = 1;
| MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
}
| IF_NOT_TYPE REG2w, IS_OBJECT_EX, >4
| GET_ZVAL_PTR REG2, prop_addr, TMP1
| GC_ADDREF REG2, TMP1w
| SET_ZVAL_PTR res_addr, REG2, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX, TMP1w, TMP2
| b >9
|.cold_code
|4:
| ldr REG0w, [FCARG1x, #(prop_info->offset + offsetof(zval, u2.extra))]
| TST_32_WITH_CONST REG0w, IS_PROP_REINITABLE, TMP1w
| beq >6
| and REG0w, REG0w, #(~IS_PROP_REINITABLE)
| str REG0w, [FCARG1x, #(prop_info->offset + offsetof(zval, u2.extra))]
| b >4
|6:
| LOAD_ADDR FCARG1x, prop_info
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_readonly_property_modification_error, REG0
| SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2
| b >9
|.code
|4:
}
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
&& ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
if (flags == ZEND_FETCH_DIM_WRITE) {
if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) {
if (!type_loaded) {
type_loaded = 1;
| MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
}
| cmp REG2w, #IS_FALSE
| ble >1
|.cold_code
|1:
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
| LOAD_ADDR FCARG2x, prop_info
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_jit_check_array_promotion, REG0
| b >9
|.code
}
} else if (flags == ZEND_FETCH_REF) {
if (!type_loaded) {
type_loaded = 1;
| MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
}
| GET_LOW_8BITS TMP1w, REG2w
| IF_TYPE TMP1w, IS_REFERENCE, >1
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2x, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
| ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
| MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
}
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
| LOAD_ZVAL_ADDR CARG3, res_addr
| EXT_CALL zend_jit_create_typed_ref, REG0
| b >9
|1:
} else {
ZEND_UNREACHABLE();
}
}
}
if (opline->opcode == ZEND_FETCH_OBJ_W) {
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
| SET_ZVAL_PTR res_addr, FCARG1x, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT, TMP1w, TMP2
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
ssa->var_info[ssa_op->result_def].indirect_reference = 1;
}
} else {
bool result_avoid_refcounting = 0;
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) {
uint32_t flags = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
const void *exit_addr;
uint32_t type;
zend_jit_addr val_addr = prop_addr;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !delayed_fetch_this
&& !op1_avoid_refcounting) {
flags = ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
type = concrete_type(res_info);
if (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| LOAD_ZVAL_ADDR REG0, prop_addr
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack,
EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (!type_loaded) {
type_loaded = 1;
| MEM_ACCESS_32_WITH_UOFFSET ldr, REG2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.type_info)), TMP1
}
| // ZVAL_DEREF()
| GET_LOW_8BITS TMP1w, REG2w
| IF_NOT_TYPE TMP1w, IS_REFERENCE, >1
| GET_Z_PTR REG0, REG0
| add REG0, REG0, #offsetof(zend_reference, val)
| GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
if (type < IS_STRING) {
|1:
if (type_loaded) {
| IF_NOT_TYPE REG2w, type, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr, ZREG_TMP1
}
} else {
if (!type_loaded) {
type_loaded = 1;
| GET_ZVAL_TYPE_INFO REG2w, val_addr, TMP1
}
|1:
| GET_LOW_8BITS TMP1w, REG2w
| IF_NOT_TYPE TMP1w, type, &exit_addr
}
| // ZVAL_COPY
| ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0
if (type < IS_STRING) {
if (Z_REG(res_addr) != ZREG_FP ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
| SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2
}
} else {
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1
if (!result_avoid_refcounting) {
| TRY_ADDREF res_info, REG2w, REG1, TMP1w
}
}
} else {
if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_REG2)) {
return 0;
}
}
}
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack,
EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
|.cold_code
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
|5:
| SET_EX_OPLINE opline, REG0
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| EXT_CALL zend_jit_fetch_obj_w_slow, REG0
} else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| EXT_CALL zend_jit_fetch_obj_r_slow, REG0
} else {
| EXT_CALL zend_jit_fetch_obj_is_slow, REG0
}
| b >9
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
|7:
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| SET_EX_OPLINE opline, REG0
if (opline->opcode != ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_UNDEF)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
if (op1_info & MAY_BE_ANY) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1, ZREG_TMP1
}
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
|1:
| LOAD_ZVAL_ADDR FCARG1x, orig_op1_addr
} else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| LOAD_ADDR FCARG2x, Z_STRVAL_P(member)
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| EXT_CALL zend_jit_invalid_property_write, REG0
| SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR, TMP1w, TMP2
} else {
| EXT_CALL zend_jit_invalid_property_read, REG0
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
}
| b >9
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
| b >9
}
}
if (!prop_info
&& may_be_dynamic
&& opline->opcode != ZEND_FETCH_OBJ_W) {
|8:
| mov FCARG2x, REG0
| SET_EX_OPLINE opline, REG0
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| EXT_CALL zend_jit_fetch_obj_r_dynamic, REG0
} else {
| EXT_CALL zend_jit_fetch_obj_is_dynamic, REG0
}
| b >9
}
|.code;
|9: // END
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_RC1)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
| IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1, ZREG_TMP1, ZREG_TMP2
| GET_ZVAL_PTR FCARG1x, orig_op1_addr, TMP1
| GC_DELREF FCARG1x, TMP1w
| bne >1
| SET_EX_OPLINE opline, REG0
| EXT_CALL zend_jit_extract_helper, REG0
|1:
} else if (!op1_avoid_refcounting) {
if (on_this) {
op1_info &= ~MAY_BE_RC1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& prop_info
&& (opline->opcode != ZEND_FETCH_OBJ_W ||
!(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
!ZEND_TYPE_IS_SET(prop_info->type))
&& (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
may_throw = 0;
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_incdec_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr res_addr = 0;
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool use_prop_guard = 0;
bool may_throw = 0;
uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
if (opline->result_type != IS_UNUSED) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1x, this_addr, TMP1
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
| GET_Z_PTR FCARG1x, FCARG1x
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
|.cold_code
|1:
| SET_EX_OPLINE opline, REG0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| LOAD_ADDR FCARG2x, ZSTR_VAL(name)
| EXT_CALL zend_jit_invalid_property_incdec, REG0
| b ->exception_handler
|.code
}
}
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
needs_slow_path = 1;
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
| cmp REG2, TMP1
| bne >7
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1
| cbnz TMP1, >7
}
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1
| tst REG0, REG0
| blt >7
| add TMP1, FCARG1x, REG0
if (!use_prop_guard) {
| ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)]
| IF_TYPE TMP2w , IS_UNDEF, >7
}
| mov FCARG1x, TMP1
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
| IF_TYPE TMP2w, IS_UNDEF, &exit_addr
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
| IF_TYPE TMP2w, IS_UNDEF, >7
needs_slow_path = 1;
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
may_throw = 1;
| SET_EX_OPLINE opline, REG0
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2x, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
| ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
| MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
}
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
if (opline->result_type == IS_UNUSED) {
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_inc_typed_prop, REG0
break;
case ZEND_PRE_DEC_OBJ:
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_dec_typed_prop, REG0
break;
default:
ZEND_UNREACHABLE();
}
} else {
| LOAD_ZVAL_ADDR CARG3, res_addr
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_typed_prop, REG0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_typed_prop, REG0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_typed_prop, REG0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_typed_prop, REG0
break;
default:
ZEND_UNREACHABLE();
}
}
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
zend_jit_addr var_addr = prop_addr;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
may_throw = 1;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1
| GET_ZVAL_PTR FCARG1x, var_addr, TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
| cbnz TMP1, >1
| add FCARG1x, FCARG1x, #offsetof(zend_reference, val)
|.cold_code
|1:
if (opline) {
| SET_EX_OPLINE opline, REG0
}
if (opline->result_type == IS_UNUSED) {
| mov FCARG2x, xzr
} else {
| LOAD_ZVAL_ADDR FCARG2x, res_addr
}
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_typed_ref, REG0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_typed_ref, REG0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_typed_ref, REG0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_typed_ref, REG0
break;
default:
ZEND_UNREACHABLE();
}
| b >9
|.code
|2:
}
if (var_info & MAY_BE_LONG) {
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2, ZREG_TMP1
}
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG1, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
| LONG_ADD_SUB_WITH_IMM adds, var_addr, Z_L(1), TMP1, TMP2
} else {
| LONG_ADD_SUB_WITH_IMM subs, var_addr, Z_L(1), TMP1, TMP2
}
| bvs >3
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
}
}
|.cold_code
}
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
may_throw = 1;
}
if (var_info & MAY_BE_LONG) {
|2:
}
if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
}
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG2, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF MAY_BE_ANY, REG0w, REG2, TMP1w
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2x, res_addr
| EXT_CALL zend_jit_pre_inc, REG0
} else {
| EXT_CALL increment_function, REG0
}
} else {
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2x, res_addr
| EXT_CALL zend_jit_pre_dec, REG0
} else {
| EXT_CALL decrement_function, REG0
}
}
if (var_info & MAY_BE_LONG) {
| b >4
}
}
if (var_info & MAY_BE_LONG) {
|3:
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
uint64_t val = 0x43e0000000000000;
| LOAD_64BIT_VAL TMP2, val
| SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
}
} else {
uint64_t val = 0xc3e0000000000000;
| LOAD_64BIT_VAL TMP2, val
| SET_ZVAL_LVAL_FROM_REG var_addr, TMP2, TMP1
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE, TMP1w, TMP2
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL_FROM_REG res_addr, TMP2, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE, TMP1w, TMP2
}
}
if (opline->result_type != IS_UNUSED
&& (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
&& prop_info
&& !ZEND_TYPE_IS_SET(prop_info->type)
&& (res_info & MAY_BE_GUARD)
&& (res_info & MAY_BE_LONG)) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
int32_t exit_point;
const void *exit_addr;
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
| b &exit_addr
|.code
} else {
| b >4
|.code
|4:
}
}
}
if (needs_slow_path) {
may_throw = 1;
|.cold_code
|7:
| SET_EX_OPLINE opline, REG0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2x, name
| ldr CARG3, EX->run_time_cache
| ADD_SUB_64_WITH_CONST_32 add, CARG3, CARG3, opline->extended_value, TMP1
if (opline->result_type == IS_UNUSED) {
| mov CARG4, xzr
} else {
| LOAD_ZVAL_ADDR CARG4, res_addr
}
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_obj_helper, REG0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_obj_helper, REG0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_obj_helper, REG0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_obj_helper, REG0
break;
default:
ZEND_UNREACHABLE();
}
| b >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_assign_obj_op(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_ssa_range *val_range,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr val_addr = OP1_DATA_ADDR();
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool use_prop_guard = 0;
bool may_throw = 0;
binary_op_type binary_op = get_binary_op(opline->extended_value);
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
ZEND_ASSERT(opline->result_type == IS_UNUSED);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1x, this_addr, TMP1
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
| GET_Z_PTR FCARG1x, FCARG1x
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
|.cold_code
|1:
| SET_EX_OPLINE opline, REG0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| LOAD_ADDR FCARG2x, ZSTR_VAL(name)
if (op1_info & MAY_BE_UNDEF) {
| EXT_CALL zend_jit_invalid_property_assign_op, REG0
} else {
| EXT_CALL zend_jit_invalid_property_assign, REG0
}
may_throw = 1;
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| b >8
} else {
| b >9
}
|.code
}
}
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
needs_slow_path = 1;
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, ((opline+1)->extended_value), TMP1
| ldr TMP2, [FCARG1x, #offsetof(zend_object, ce)]
| cmp REG2, TMP2
| bne >7
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, TMP1, REG0, ((opline+1)->extended_value + sizeof(void*) * 2), TMP1
| cbnz TMP1, >7
}
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, ((opline+1)->extended_value + sizeof(void*)), TMP1
| tst REG0, REG0
| blt >7
| add TMP1, FCARG1x, REG0
if (!use_prop_guard) {
| ldrb TMP2w, [TMP1, #offsetof(zval,u1.v.type)]
| IF_TYPE TMP2w, IS_UNDEF, >7
}
| mov FCARG1x, TMP1
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
| IF_TYPE TMP2w, IS_UNDEF, &exit_addr
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
| IF_TYPE TMP2w, IS_UNDEF, >7
needs_slow_path = 1;
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t info = val_info;
may_throw = 1;
if (opline) {
| SET_EX_OPLINE opline, REG0
}
| IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1, ZREG_TMP1
|.cold_code
|1:
| GET_ZVAL_PTR FCARG1x, prop_addr, TMP1
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2x, val_addr
}
| LOAD_ADDR CARG3, binary_op
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
}
| b >9
|.code
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2x, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
| ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
| MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
}
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
| LOAD_ZVAL_ADDR CARG3, val_addr
| LOAD_ADDR CARG4, binary_op
| EXT_CALL zend_jit_assign_op_to_typed_prop, REG0
if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
info |= MAY_BE_RC1|MAY_BE_RCN;
}
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
zend_jit_addr var_addr = prop_addr;
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr, ZREG_TMP1
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
may_throw = 1;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
| LOAD_ZVAL_ADDR REG0, prop_addr
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2, ZREG_TMP1
| GET_ZVAL_PTR FCARG1x, var_addr, TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_reference, sources.ptr)]
| cbnz TMP1, >1
| add REG0, FCARG1x, #offsetof(zend_reference, val)
|.cold_code
|1:
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2x, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, REG0
}
| LOAD_ADDR CARG3, binary_op
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, REG0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, REG0
}
| b >9
|.code
|2:
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, 0)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
may_throw = 1;
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
(val_info & MAY_BE_ANY) != MAY_BE_STRING) {
may_throw = 1;
}
}
goto long_math;
case ZEND_SL:
case ZEND_SR:
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
}
if ((opline+1)->op1_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
may_throw = 1;
}
goto long_math;
case ZEND_MOD:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if ((opline+1)->op1_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
may_throw = 1;
}
long_math:
if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
(opline+1)->op1_type, (opline+1)->op1, val_addr, val_info,
val_range,
0, var_addr, var_def_info, var_info, 0)) {
return 0;
}
break;
case ZEND_CONCAT:
may_throw = 1;
if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr,
0)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (needs_slow_path) {
may_throw = 1;
|.cold_code
|7:
| SET_EX_OPLINE opline, REG0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2x, name
| LOAD_ZVAL_ADDR CARG3, val_addr
| ldr CARG4, EX->run_time_cache
| ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, (opline+1)->extended_value, TMP1
| LOAD_ADDR CARG5, binary_op
| EXT_CALL zend_jit_assign_obj_op_helper, REG0
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
|8:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2
| b >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_assign_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr val_addr = OP1_DATA_ADDR();
zend_jit_addr res_addr = 0;
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool needs_val_dtor = 0;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1x, this_addr, TMP1
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
| IF_NOT_Z_TYPE FCARG1x, IS_INDIRECT, >1, TMP1w
| GET_Z_PTR FCARG1x, FCARG1x
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1, ZREG_TMP1
|.cold_code
|1:
| SET_EX_OPLINE opline, REG0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
}
| LOAD_ADDR FCARG2x, ZSTR_VAL(name)
| EXT_CALL zend_jit_invalid_property_assign, REG0
if (RETURN_VALUE_USED(opline)) {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL, TMP1w, TMP2
}
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
needs_val_dtor = 1;
| b >7
} else {
| b >9
}
|.code
}
}
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
needs_slow_path = 1;
| ldr REG0, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG2, REG0, opline->extended_value, TMP1
| ldr TMP1, [FCARG1x, #offsetof(zend_object, ce)]
| cmp REG2, TMP1
| bne >5
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, (opline->extended_value + sizeof(void*) * 2), TMP1
}
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, REG0, (opline->extended_value + sizeof(void*)), TMP1
| tst REG0, REG0
| blt >5
| add TMP2, FCARG1x, REG0
| ldrb TMP1w, [TMP2, #offsetof(zval,u1.v.type)]
| IF_TYPE TMP1w, IS_UNDEF, >5
| mov FCARG1x, TMP2
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| cbnz FCARG2x, >1
|.cold_code
|1:
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
| SET_EX_OPLINE opline, REG0
| LOAD_ZVAL_ADDR CARG3, val_addr
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR CARG4, res_addr
} else {
| mov CARG4, xzr
}
| EXT_CALL zend_jit_assign_to_typed_prop, REG0
if ((opline+1)->op1_type == IS_CONST) {
| // TODO: ???
| // if (Z_TYPE_P(value) == orig_type) {
| // CACHE_PTR_EX(cache_slot + 2, NULL);
}
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| b >7
} else {
| b >9
}
|.code
}
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) {
// Undefined property with magic __get()/__set()
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
| IF_TYPE TMP2w, IS_UNDEF, &exit_addr
} else {
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP2w, FCARG1x, (prop_info->offset + offsetof(zval,u1.v.type)), TMP1
| IF_TYPE TMP2w, IS_UNDEF, >5
needs_slow_path = 1;
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t info = val_info;
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
| SET_EX_OPLINE opline, REG0
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2x, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| ldr REG0, [FCARG1x, #offsetof(zend_object, ce)]
| ldr REG0, [REG0, #offsetof(zend_class_entry, properties_info_table)]
| MEM_ACCESS_64_WITH_UOFFSET ldr, FCARG2x, REG0, prop_info_offset, TMP1
}
| LOAD_ZVAL_ADDR FCARG1x, prop_addr
| LOAD_ZVAL_ADDR CARG3, val_addr
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR CARG4, res_addr
} else {
| mov CARG4, xzr
}
| EXT_CALL zend_jit_assign_to_typed_prop, REG0
if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
info |= MAY_BE_RC1|MAY_BE_RCN;
}
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL, ZREG_TMP1, ZREG_TMP2
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
// value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES());
if (opline->result_type == IS_UNUSED) {
if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
}
}
if (needs_slow_path) {
|.cold_code
|5:
| SET_EX_OPLINE opline, REG0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2x, name
| LOAD_ZVAL_ADDR CARG3, val_addr
| ldr CARG4, EX->run_time_cache
| ADD_SUB_64_WITH_CONST_32 add, CARG4, CARG4, opline->extended_value, TMP1
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR CARG5, res_addr
} else {
| mov CARG5, xzr
}
| EXT_CALL zend_jit_assign_obj_helper, REG0
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
|7:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2
| b >9
|.code
} else if (needs_val_dtor) {
|.cold_code
|7:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline, ZREG_TMP1, ZREG_TMP2
| b >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline, ZREG_TMP1, ZREG_TMP2
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw)
{
zend_jit_addr op1_addr = OP1_ADDR();
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (may_throw) {
| SET_EX_OPLINE opline, REG0
}
if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
if (op1_info & MAY_BE_ARRAY) {
| IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7, ZREG_TMP1
}
| MEM_ACCESS_32_WITH_UOFFSET ldr, FCARG1w, FP, (opline->op1.var + offsetof(zval, u2.fe_iter_idx)), TMP1
| mvn TMP1w, wzr // TODO: DynAsm fails loading #-1
| cmp FCARG1w, TMP1w
| beq >7
| EXT_CALL zend_hash_iterator_del, REG0
|7:
}
| ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
}
return 1;
}
static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
if (len > 0) {
const char *str = Z_STRVAL_P(zv);
| SET_EX_OPLINE opline, REG0
| LOAD_ADDR CARG1, str
| LOAD_64BIT_VAL CARG2, len
| EXT_CALL zend_write, REG0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
} else {
zend_jit_addr op1_addr = OP1_ADDR();
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
| SET_EX_OPLINE opline, REG0
| GET_ZVAL_PTR REG0, op1_addr, TMP1
| add CARG1, REG0, #offsetof(zend_string, val)
| ldr CARG2, [REG0, #offsetof(zend_string, len)]
| EXT_CALL zend_write, REG0
if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
| ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline, ZREG_TMP1, ZREG_TMP2
}
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
| SET_ZVAL_LVAL res_addr, len, TMP1, TMP2
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1
| ldr Rx(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(zend_string, len)]
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
| GET_ZVAL_PTR REG0, op1_addr, TMP1
| ldr REG0, [REG0, #offsetof(zend_string, len)]
| SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
}
return 1;
}
static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
zend_long count;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
count = zend_hash_num_elements(Z_ARRVAL_P(zv));
| SET_ZVAL_LVAL res_addr, count, TMP1, TMP2
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
// Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.
if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_PTR Rx(Z_REG(res_addr)), op1_addr, TMP1
// Sign-extend the 32-bit value to a potentially 64-bit zend_long
| ldr Rw(Z_REG(res_addr)), [Rx(Z_REG(res_addr)), #offsetof(HashTable, nNumOfElements)]
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
| GET_ZVAL_PTR REG0, op1_addr, TMP1
// Sign-extend the 32-bit value to a potentially 64-bit zend_long
| ldr REG0w, [REG0, #offsetof(HashTable, nNumOfElements)]
| SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline, ZREG_TMP1, ZREG_TMP2
}
if (may_throw) {
return zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_load_this(dasm_State **Dst, uint32_t var)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
| ldr FCARG1x, EX->This.value.ptr
| SET_ZVAL_PTR var_addr, FCARG1x, TMP1
| SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX, TMP1w, TMP2
| GC_ADDREF FCARG1x, TMP1w
return 1;
}
static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only)
{
if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
!TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| ldrb TMP1w, EX->This.u1.v.type
| cmp TMP1w, #IS_OBJECT
| bne &exit_addr
if (JIT_G(current_frame)) {
TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
}
}
} else {
| ldrb TMP1w, EX->This.u1.v.type
| cmp TMP1w, #IS_OBJECT
| bne >1
|.cold_code
|1:
| SET_EX_OPLINE opline, REG0
| b ->invalid_this
|.code
}
}
if (!check_only) {
if (!zend_jit_load_this(Dst, opline->result.var)) {
return 0;
}
}
return 1;
}
static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info)
{
uint32_t count;
Bucket *p;
const zend_op *target;
int b;
int32_t exit_point;
const void *exit_addr;
if (default_label) {
| cbz REG0, &default_label
} else if (next_opline) {
| cbz REG0, >3
} else {
| cbz REG0, =>default_b
}
| LOAD_ADDR FCARG1x, jumptable
| ldr TMP1, [FCARG1x, #offsetof(HashTable, arData)]
| sub REG0, REG0, TMP1
if (HT_IS_PACKED(jumptable)) {
| mov FCARG1x, #(sizeof(zval) / sizeof(void*))
} else {
| mov FCARG1x, #(sizeof(Bucket) / sizeof(void*))
}
| sdiv REG0, REG0, FCARG1x
| adr FCARG1x, >4
| ldr TMP1, [FCARG1x, REG0]
| br TMP1
|.jmp_table
|.align 8
|4:
if (trace_info) {
trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
}
count = jumptable->nNumUsed;
p = jumptable->arData;
do {
if (Z_TYPE(p->val) == IS_UNDEF) {
if (default_label) {
| .addr &default_label
} else if (next_opline) {
| .addr >3
} else {
| .addr =>default_b
}
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val));
if (!next_opline) {
b = ssa->cfg.map[target - op_array->opcodes];
| .addr =>b
} else if (next_opline == target) {
| .addr >3
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| .addr &exit_addr
}
}
if (HT_IS_PACKED(jumptable)) {
p = (Bucket*)(((zval*)p)+1);
} else {
p++;
}
count--;
} while (count);
|.code
return 1;
}
static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
{
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *next_opline = NULL;
if (trace) {
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
ZEND_ASSERT(trace->opline != NULL);
next_opline = trace->opline;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
zval *jump_zv = NULL;
int b;
if (opline->opcode == ZEND_SWITCH_LONG) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else if (opline->opcode == ZEND_MATCH) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
} else if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else {
ZEND_UNREACHABLE();
}
if (next_opline) {
const zend_op *target;
if (jump_zv != NULL) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
}
ZEND_ASSERT(target == next_opline);
} else {
if (jump_zv != NULL) {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
} else {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
}
| b =>b
}
} else {
zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
uint32_t op1_info = OP1_INFO();
zend_jit_addr op1_addr = OP1_ADDR();
const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
const zend_op *target;
int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
int b;
int32_t exit_point;
const void *fallback_label = NULL;
const void *default_label = NULL;
const void *exit_addr;
if (next_opline) {
if (next_opline != opline + 1) {
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
fallback_label = zend_jit_trace_get_exit_addr(exit_point);
if (!fallback_label) {
return 0;
}
}
if (next_opline != default_opline) {
exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
default_label = zend_jit_trace_get_exit_addr(exit_point);
if (!default_label) {
return 0;
}
}
}
if (opline->opcode == ZEND_SWITCH_LONG) {
if (op1_info & MAY_BE_LONG) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1, ZREG_TMP1
| GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1
|.cold_code
|1:
| // ZVAL_DEREF(op)
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1
}
| GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
if (fallback_label) {
| add TMP1, FCARG2x, #offsetof(zend_reference, val)
| IF_NOT_Z_TYPE TMP1, IS_LONG, &fallback_label, TMP2w
} else {
| add TMP1, FCARG2x, #offsetof(zend_reference, val)
| IF_NOT_Z_TYPE TMP1, IS_LONG, >3, TMP2w
}
| ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.lval)]
| b >2
|.code
|2:
} else {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1
}
}
| GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1
}
if (HT_IS_PACKED(jumptable)) {
uint32_t count = jumptable->nNumUsed;
zval *zv = jumptable->arPacked;
| CMP_64_WITH_CONST_32 FCARG2x, jumptable->nNumUsed, TMP1
if (default_label) {
| bhs &default_label
} else if (next_opline) {
| bhs >3
} else {
| bhs =>default_b
}
| adr REG0, >4
| ldr TMP1, [REG0, FCARG2x, lsl #3]
| br TMP1
|.jmp_table
|.align 8
|4:
if (trace_info) {
trace_info->jmp_table_size += count;
}
do {
if (Z_TYPE_P(zv) == IS_UNDEF) {
if (default_label) {
| .addr &default_label
} else if (next_opline) {
| .addr >3
} else {
| .addr =>default_b
}
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (!next_opline) {
b = ssa->cfg.map[target - op_array->opcodes];
| .addr =>b
} else if (next_opline == target) {
| .addr >3
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| .addr &exit_addr
}
}
zv++;
count--;
} while (count);
|.code
|3:
} else {
| LOAD_ADDR FCARG1x, jumptable
| EXT_CALL zend_hash_index_find, REG0
| mov REG0, RETVALx
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
|3:
}
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (op1_info & MAY_BE_STRING) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1, ZREG_TMP1
| GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
|.cold_code
|1:
| // ZVAL_DEREF(op)
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3, ZREG_TMP1
}
| GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
if (fallback_label) {
| add TMP1, FCARG2x, #offsetof(zend_reference, val)
| IF_NOT_Z_TYPE TMP1, IS_STRING, &fallback_label, TMP2w
} else {
| add TMP1, FCARG2x, #offsetof(zend_reference, val)
| IF_NOT_Z_TYPE TMP1, IS_STRING, >3, TMP2w
}
| ldr FCARG2x, [FCARG2x, #offsetof(zend_reference, val.value.ptr)]
| b >2
|.code
|2:
} else {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1
}
}
| GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
}
| LOAD_ADDR FCARG1x, jumptable
| EXT_CALL zend_hash_find, REG0
| mov REG0, RETVALx
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
|3:
}
} else if (opline->opcode == ZEND_MATCH) {
if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG2x, op1_addr
| ZVAL_DEREF FCARG2x, op1_info, TMP1w
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
}
| LOAD_ADDR FCARG1x, jumptable
if (op1_info & MAY_BE_LONG) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & MAY_BE_STRING) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5, ZREG_TMP1
} else if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6, ZREG_TMP1
} else if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label, ZREG_TMP1
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b, ZREG_TMP1
}
}
| GET_ZVAL_LVAL ZREG_FCARG2, op1_addr, TMP1
| EXT_CALL zend_hash_index_find, REG0
| mov REG0, RETVALx
if (op1_info & MAY_BE_STRING) {
| b >2
}
}
if (op1_info & MAY_BE_STRING) {
|5:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6, ZREG_TMP1
} else if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label, ZREG_TMP1
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b, ZREG_TMP1
}
}
| GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
| EXT_CALL zend_hash_find, REG0
| mov REG0, RETVALx
}
|2:
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
}
if (op1_info & MAY_BE_UNDEF) {
|6:
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label, ZREG_TMP1
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3, ZREG_TMP1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b, ZREG_TMP1
}
}
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, REG0
| LOAD_32BIT_VAL FCARG1w, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (default_label) {
| b &default_label
} else if (next_opline) {
| b >3
} else {
| b =>default_b
}
|3:
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
{
zend_arg_info *arg_info = &op_array->arg_info[-1];
ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
zend_jit_addr op1_addr = OP1_ADDR();
bool needs_slow_check = 1;
bool slow_check_in_cold = 1;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
if (type_mask == 0) {
slow_check_in_cold = 0;
} else {
if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
slow_check_in_cold = 0;
} else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
needs_slow_check = 0;
} else if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
| IF_NOT_ZVAL_TYPE op1_addr, type_code, >6, ZREG_TMP1
} else {
| mov REG2w, #1
| GET_ZVAL_TYPE REG1w, op1_addr, TMP1
| lsl REG2w, REG2w, REG1w
| TST_32_WITH_CONST REG2w, type_mask, TMP1w
| beq >6
}
}
if (needs_slow_check) {
if (slow_check_in_cold) {
|.cold_code
|6:
}
| SET_EX_OPLINE opline, REG1
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7, ZREG_TMP1
| LOAD_32BIT_VAL FCARG1x, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, REG0
| cbz RETVALx, ->exception_handler
| LOAD_ADDR_ZTS FCARG1x, executor_globals, uninitialized_zval
| b >8
}
|7:
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
|8:
| ldr FCARG2x, EX->func
| LOAD_ADDR CARG3, (ptrdiff_t)arg_info
| ldr REG0, EX->run_time_cache
| ADD_SUB_64_WITH_CONST_32 add, CARG4, REG0, opline->op2.num, TMP1
| EXT_CALL zend_jit_verify_return_slow, REG0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
if (slow_check_in_cold) {
| b >9
|.code
}
}
|9:
return 1;
}
static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
if (op1_info & MAY_BE_REF) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, op1_addr
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
| ZVAL_DEREF FCARG1x, op1_info, TMP1w
|1:
}
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| b =>target_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE, TMP1w, TMP2
}
} else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
} else if (smart_branch_opcode) {
if (smart_branch_opcode != ZEND_JMPNZ) {
| b =>target_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE, TMP1w, TMP2
}
} else {
ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL);
| MEM_ACCESS_8_WITH_UOFFSET ldrb, TMP1w, Rx(Z_REG(op1_addr)), (Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)), TMP1
| cmp TMP1w, #IS_NULL
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| bgt &exit_addr
} else {
| ble &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| ble =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| bgt =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
| cset REG0w, gt
| add REG0w, REG0w, #IS_FALSE
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
}
return 1;
}
static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_REG0, ZREG_TMP1, ZREG_FPR0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, REG0, TMP1
}
} else {
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
| // ZVAL_COPY(res, value);
| ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
if (opline->op1_type == IS_CV) {
| TRY_ADDREF op1_info, REG0w, FCARG1x, TMP1w
}
}
| // Z_FE_POS_P(res) = 0;
| MEM_ACCESS_32_WITH_UOFFSET str, wzr, FP, (opline->result.var + offsetof(zval, u2.fe_pos)), TMP1
return 1;
}
static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr)
{
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) {
/* empty array */
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
| b &exit_addr
}
} else {
| b =>target_label
}
return 1;
}
| // array = EX_VAR(opline->op1.var);
| // fe_ht = Z_ARRVAL_P(array);
| GET_ZVAL_PTR FCARG1x, op1_addr, TMP1
if (op1_info & MAY_BE_PACKED_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| beq &exit_addr
} else {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| bne &exit_addr
}
}
| // pos = Z_FE_POS_P(array);
| MEM_ACCESS_32_WITH_UOFFSET ldr, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1
if (MAY_BE_HASH(op1_info)) {
if (MAY_BE_PACKED(op1_info)) {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, u.flags)]
| TST_32_WITH_CONST TMP1w, HASH_FLAG_PACKED, TMP2w
| bne >2
}
| // p = fe_ht->arData + pos;
|| ZEND_ASSERT(sizeof(Bucket) == 32);
| mov FCARG2w, REG0w
| ldr TMP1, [FCARG1x, #offsetof(zend_array, arData)]
| add FCARG2x, TMP1, FCARG2x, lsl #5
|1:
| // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)]
| cmp TMP1w, REG0w
| // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
| // ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
| bls &exit_addr
} else {
| bls >3
}
} else {
| bls =>target_label
}
| // pos++;
| add REG0w, REG0w, #1
| // value_type = Z_TYPE_INFO_P(value);
| // if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
| IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >3, TMP1w
} else {
| IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w
}
| // p++;
| add FCARG2x, FCARG2x, #sizeof(Bucket)
| b <1
if (MAY_BE_PACKED(op1_info)) {
|2:
}
}
if (MAY_BE_PACKED(op1_info)) {
| // p = fe_ht->arPacked + pos;
|| ZEND_ASSERT(sizeof(zval) == 16);
| mov FCARG2w, REG0w
| ldr TMP1, [FCARG1x, #offsetof(zend_array, arPacked)]
| add FCARG2x, TMP1, FCARG2x, lsl #4
|1:
| // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
| ldr TMP1w, [FCARG1x, #offsetof(zend_array, nNumUsed)]
| cmp TMP1w, REG0w
| // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
| // ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
| bls &exit_addr
} else {
| bls >4
}
} else {
| bls =>target_label
}
| // pos++;
| add REG0w, REG0w, #1
| // value_type = Z_TYPE_INFO_P(value);
| // if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
| IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, >4, TMP1w
} else {
| IF_NOT_Z_TYPE FCARG2x, IS_UNDEF, &exit_addr, TMP1w
}
| // p++;
| add FCARG2x, FCARG2x, #sizeof(zval)
| b <1
}
if (!exit_addr || exit_opcode == ZEND_JMP) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
uint32_t val_info;
if (RETURN_VALUE_USED(opline)) {
zend_jit_addr res_addr = RES_ADDR();
if (MAY_BE_HASH(op1_info)) {
|3:
| // Z_FE_POS_P(array) = pos + 1;
| MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1
if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
&& (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
| // if (!p->key) {
| ldr REG0, [FCARG2x, #offsetof(Bucket, key)]
| cbz REG0, >2
}
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
| // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
| ldr REG0, [FCARG2x, #offsetof(Bucket, key)]
| SET_ZVAL_PTR res_addr, REG0, TMP1
| ldr TMP1w, [REG0, #offsetof(zend_refcounted, gc.u.type_info)]
| TST_32_WITH_CONST TMP1w, IS_STR_INTERNED, TMP2w
| beq >1
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING, TMP1w, TMP2
| b >3
|1:
| GC_ADDREF REG0, TMP1w
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2
if ((op1_info & MAY_BE_ARRAY_KEY_LONG) || MAY_BE_PACKED(op1_info)) {
| b >3
|2:
}
}
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
| // ZVAL_LONG(EX_VAR(opline->result.var), p->h);
| ldr REG0, [FCARG2x, #offsetof(Bucket, h)]
| SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
if (MAY_BE_PACKED(op1_info)) {
| b >3
}
}
}
if (MAY_BE_PACKED(op1_info)) {
|4:
| // Z_FE_POS_P(array) = pos + 1;
| MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1
| sub REG0w, REG0w, #1
| SET_ZVAL_LVAL_FROM_REG res_addr, REG0, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG, TMP1w, TMP2
}
|3:
} else {
|3:
|4:
| // Z_FE_POS_P(array) = pos + 1;
| MEM_ACCESS_32_WITH_UOFFSET str, REG0w, FP, (opline->op1.var + offsetof(zval, u2.fe_pos)), TMP1
}
val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
if (val_info & MAY_BE_ARRAY) {
val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
}
if (op1_info & MAY_BE_ARRAY_OF_REF) {
val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
} else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1 | MAY_BE_RCN;
}
if (opline->op2_type == IS_CV) {
| // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) {
return 0;
}
} else {
| // ZVAL_COPY(res, value);
| ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_REG0, ZREG_FCARG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF val_info, REG0w, FCARG1x, TMP1w
}
} else {
|3:
|4:
}
return 1;
}
static int zend_jit_fetch_constant(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
zend_jit_addr res_addr)
{
zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0);
uint32_t res_info = RES_INFO();
| // c = CACHED_PTR(opline->extended_value);
| ldr FCARG1x, EX->run_time_cache
| MEM_ACCESS_64_WITH_UOFFSET ldr, REG0, FCARG1x, opline->extended_value, TMP1
| // if (c != NULL)
| cbz REG0, >9
if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
| // if (!IS_SPECIAL_CACHE_VAL(c))
|| ZEND_ASSERT(CACHE_SPECIAL == 1);
| TST_64_WITH_ONE REG0
| bne >9
}
|8:
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
int32_t exit_point;
const void *exit_addr = NULL;
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
exit_point = zend_jit_trace_get_exit_point(opline+1, 0);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
uint32_t type = concrete_type(res_info);
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr, ZREG_TMP1
} else {
| GET_ZVAL_TYPE_INFO REG2w, const_addr, TMP1
| IF_NOT_TYPE REG2w, type, &exit_addr
}
| ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_FPR0
if (type < IS_STRING) {
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, type, TMP1w, TMP2
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
} else {
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG2w, TMP1
| TRY_ADDREF res_info, REG2w, REG1, TMP1w
}
} else {
| ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_REG0, ZREG_REG1, ZREG_TMP1, ZREG_TMP2, ZREG_FPR0
| TRY_ADDREF MAY_BE_ANY, REG0w, REG1, TMP1w
}
|.cold_code
|9:
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, REG0
| // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC);
| LOAD_ADDR FCARG1x, zv
| LOAD_32BIT_VAL FCARG2w, opline->op1.num
| EXT_CALL zend_jit_get_constant, REG0
| mov REG0, RETVALx
| // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION();
| cbnz REG0, <8
| b ->exception_handler
|.code
return 1;
}
static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
| // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
| LOAD_ADDR FCARG1x, ht
if (opline->op1_type != IS_CONST) {
| GET_ZVAL_PTR FCARG2x, op1_addr, TMP1
| EXT_CALL zend_hash_find, REG0
} else {
zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
| LOAD_ADDR FCARG2x, str
| EXT_CALL zend_hash_find_known_hash, REG0
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| cbz RETVALx, &exit_addr
} else {
| cbnz RETVALx, &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| cbz RETVALx, =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| cbnz RETVALx, =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
| tst RETVALx, RETVALx
| cset REG0w, ne
| add REG0w, REG0w, #IS_FALSE
| SET_ZVAL_TYPE_INFO_FROM_REG res_addr, REG0w, TMP1
}
return 1;
}
static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info)
{
uint32_t offset;
offset = (opline->opcode == ZEND_ROPE_INIT) ?
opline->result.var :
opline->op1.var + opline->extended_value * sizeof(zend_string*);
if (opline->op2_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_string *str;
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
str = Z_STR_P(zv);
| LOAD_ADDR REG0, str
| MEM_ACCESS_64_WITH_UOFFSET str, REG0, FP, offset, TMP1
} else {
zend_jit_addr op2_addr = OP2_ADDR();
ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
| GET_ZVAL_PTR REG1, op2_addr, TMP1
| MEM_ACCESS_64_WITH_UOFFSET str, REG1, FP, offset, TMP1
if (opline->op2_type == IS_CV) {
| GET_ZVAL_TYPE_INFO REG0w, op2_addr, TMP1
| TRY_ADDREF op2_info, REG0w, REG1, TMP1w
}
}
if (opline->opcode == ZEND_ROPE_END) {
zend_jit_addr res_addr = RES_ADDR();
| ADD_SUB_64_WITH_CONST add, FCARG1x, FP, opline->op1.var, TMP1
| LOAD_32BIT_VAL FCARG2w, opline->extended_value
| EXT_CALL zend_jit_rope_end, TMP1
| SET_ZVAL_PTR res_addr, RETVALx, TMP1
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX, TMP1w, TMP2
}
return 1;
}
static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1
return 1;
}
static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
const void *exit_addr = NULL;
if (add_ref_guard || add_type_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (add_ref_guard) {
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr, ZREG_TMP1
}
if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
/* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1x, var_addr
}
| EXT_CALL zend_jit_unref_helper, REG0
} else {
| GET_ZVAL_PTR FCARG1x, var_addr, TMP1
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val));
*var_addr_ptr = var_addr;
}
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (add_type_guard
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
| IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr, ZREG_TMP1
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
} else {
var_info &= ~MAY_BE_REF;
*var_info_ptr = var_info;
}
*var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
return 1;
}
static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
int32_t exit_point;
const void *exit_addr;
if (add_indirect_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr, ZREG_TMP1
| GET_ZVAL_PTR FCARG1x, var_addr, TMP1
} else {
/* May be already loaded into FCARG1a or RAX by previous FETCH_OBJ_W/DIM_W */
if (opline->op1_type != IS_VAR ||
(opline-1)->result_type != IS_VAR ||
(opline-1)->result.var != opline->op1.var ||
(opline-1)->op1_type == IS_VAR ||
(opline-1)->op2_type == IS_VAR ||
(opline-1)->op2_type == IS_TMP_VAR) {
| GET_ZVAL_PTR FCARG1x, var_addr, TMP1
} else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) {
| mov FCARG1x, REG0
}
}
*var_info_ptr &= ~MAY_BE_INDIRECT;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
*var_addr_ptr = var_addr;
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (!(var_type & IS_TRACE_REFERENCE)
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_Z_TYPE FCARG1x, var_type, &exit_addr, TMP1w
//var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
}
return 1;
}
static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var)
{
if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) {
return 0;
}
switch (opline->opcode) {
case ZEND_QM_ASSIGN:
case ZEND_SEND_VAR:
case ZEND_ASSIGN:
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
return 1;
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
if (def_var == ssa_op->result_def &&
use_var == ssa_op->op1_use) {
return 1;
}
break;
default:
break;
}
return 0;
}
static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_SEND_VAR:
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
return (opline->op2_type != IS_CONST);
case ZEND_QM_ASSIGN:
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
return 1;
case ZEND_RETURN:
return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
case ZEND_ASSIGN:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return
opline->op1_type == IS_CV &&
!(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)));
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE)));
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG));
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
op1_info = OP1_INFO();
op2_info = OP1_DEF_INFO();
return opline->op1_type == IS_CV
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG))
&& (op2_info & MAY_BE_LONG);
case ZEND_STRLEN:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
case ZEND_COUNT:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
case ZEND_JMPZ:
case ZEND_JMPNZ:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (!ssa->cfg.map) {
return 0;
}
if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
return 0;
}
}
ZEND_FALLTHROUGH;
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
return 1;
case ZEND_FETCH_CONSTANT:
return 1;
case ZEND_FETCH_DIM_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace
&& trace->op1_type != IS_UNKNOWN
&& (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
}
return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
(!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) &&
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) &&
(!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1))));
}
return 0;
}
static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
{
if (ssa->vars[var].no_val) {
/* we don't need the value */
return 0;
}
if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
/* Disable global register allocation,
* register allocation for SSA variables connected through Phi functions
*/
if (ssa->vars[var].definition_phi) {
return 0;
}
if (ssa->vars[var].phi_use_chain) {
zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
do {
if (!ssa->vars[phi->ssa_var].no_val) {
return 0;
}
phi = zend_ssa_next_use_phi(ssa, var, phi);
} while (phi);
}
}
if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
/* bad type */
return 0;
}
return 1;
}
static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
{
if (!zend_jit_var_supports_reg(ssa, var)) {
return 0;
}
if (ssa->vars[var].definition >= 0) {
uint32_t def = ssa->vars[var].definition;
if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
return 0;
}
}
if (ssa->vars[var].use_chain >= 0) {
int use = ssa->vars[var].use_chain;
do {
if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
return 0;
}
use = zend_ssa_next_use(ssa->ops, var, use);
} while (use >= 0);
}
return 1;
}
static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_FETCH_DIM_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) ||
((opline->op2_type & (IS_TMP_VAR|IS_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) {
return ZEND_REGSET(ZREG_FCARG1);
}
break;
default:
break;
}
return ZEND_REGSET_EMPTY;
}
static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use)
{
uint32_t op1_info, op2_info, res_info;
zend_regset regset = ZEND_REGSET_SCRATCH;
switch (opline->opcode) {
case ZEND_NOP:
case ZEND_OP_DATA:
case ZEND_JMP:
case ZEND_RETURN:
regset = ZEND_REGSET_EMPTY;
break;
case ZEND_QM_ASSIGN:
if (ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
/* break missing intentionally */
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
if (opline->op2_type == IS_CONST) {
break;
}
if (ssa_op->op1_use == current_var) {
regset = ZEND_REGSET(ZREG_REG0);
break;
}
op1_info = OP1_INFO();
if (!(op1_info & MAY_BE_UNDEF)) {
if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_FPR0);
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
regset = ZEND_REGSET(ZREG_REG0);
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2));
}
}
break;
case ZEND_SEND_VAR:
if (opline->op2_type == IS_CONST) {
break;
}
if (ssa_op->op1_use == current_var ||
ssa_op->op1_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
if (!(op1_info & MAY_BE_UNDEF)) {
if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_FPR0);
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2));
if (op1_info & MAY_BE_REF) {
ZEND_REGSET_INCL(regset, ZREG_REG1);
}
}
}
break;
case ZEND_ASSIGN:
if (ssa_op->op2_use == current_var ||
ssa_op->op2_def == current_var ||
ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (opline->op1_type == IS_CV
&& !(op2_info & MAY_BE_UNDEF)
&& !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_FPR0);
} else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
regset = ZEND_REGSET(ZREG_REG0);
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_REG0), ZEND_REGSET(ZREG_REG2));
}
}
break;
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
if (ssa_op->op1_use == current_var ||
ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
if (opline->op1_type == IS_CV
&& (op1_info & MAY_BE_LONG)
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (op1_info & MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_FPR0);
}
if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) {
ZEND_REGSET_INCL(regset, ZREG_REG1);
}
}
break;
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
}
res_info = RES_INFO();
if (res_info & MAY_BE_DOUBLE) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
ZEND_REGSET_INCL(regset, ZREG_FPR0);
ZEND_REGSET_INCL(regset, ZREG_FPR1);
} else if (res_info & MAY_BE_GUARD) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->result_def != current_var) {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
}
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
if (zend_is_commutative(opline->opcode)) {
if (ssa_op->result_def != current_var) {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
}
} else {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_FPR1);
}
}
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use) &&
(!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
}
}
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
regset = ZEND_REGSET_EMPTY;
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
}
}
break;
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) &&
opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) {
if (ssa_op->op1_use != current_var &&
ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->op1_use != current_var &&
ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
}
}
}
break;
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
op1_info = OP1_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (op1_info & MAY_BE_DOUBLE) {
ZEND_REGSET_INCL(regset, ZREG_FPR0);
}
if (opline->opcode == ZEND_BOOL ||
opline->opcode == ZEND_BOOL_NOT ||
opline->opcode == ZEND_JMPZ_EX ||
opline->opcode == ZEND_JMPNZ_EX) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
}
}
break;
case ZEND_DO_UCALL:
case ZEND_DO_FCALL:
case ZEND_DO_FCALL_BY_NAME:
case ZEND_INCLUDE_OR_EVAL:
case ZEND_GENERATOR_CREATE:
case ZEND_YIELD:
case ZEND_YIELD_FROM:
regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
break;
default:
break;
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (ssa_op == ssa->ops
&& JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL
&& (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
ZEND_REGSET_INCL(regset, ZREG_REG0);
ZEND_REGSET_INCL(regset, ZREG_REG1);
}
}
return regset;
}
static size_t dasm_venners_size = 0;
void **dasm_labels_veneers = NULL;
static int zend_jit_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, uint32_t *cp, ptrdiff_t offset)
{
void *veneer;
ptrdiff_t na;
int n, m;
/* try to reuse veneers for global labels */
if ((ins >> 16) == DASM_REL_LG
&& *(b-1) < 0
&& dasm_labels_veneers[-*(b-1)]) {
veneer = dasm_labels_veneers[-*(b-1)];
na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
n = (int)na;
/* check if we can jump to veneer */
if ((ptrdiff_t)n != na) {
/* pass */
} else if (!(ins & 0xf800)) { /* B, BL */
if ((n & 3) == 0 && ((n+0x08000000) >> 28) == 0) {
return n;
}
} else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */
if ((n & 3) == 0 && ((n+0x00100000) >> 21) == 0) {
return n;
}
} else if ((ins & 0x3000) == 0x2000) { /* ADR */
/* pass */
} else if ((ins & 0x3000) == 0x3000) { /* ADRP */
/* pass */
} else if ((ins & 0x1000)) { /* TBZ, TBNZ */
if ((n & 3) == 0 && ((n+0x00008000) >> 16) == 0) {
return n;
}
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& (ins >> 16) == DASM_REL_A) {
ptrdiff_t addr = (((ptrdiff_t)(*(b-1))) << 32) | (unsigned int)(*(b-2));
if ((void*)addr >= dasm_buf && (void*)addr < dasm_end) {
uint32_t exit_point = zend_jit_trace_find_exit_point((void*)addr);
zend_jit_trace_info *t = zend_jit_get_current_trace_info();
if (exit_point != (uint32_t)-1) {
/* Use exit points table */
ZEND_ASSERT(exit_point < t->exit_count);
veneer = (char*)buffer + dasm_getpclabel(&Dst, 1) - (t->exit_count - exit_point) * 4;
na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
n = (int)na;
/* check if we can jump to veneer */
if ((ptrdiff_t)n != na) {
ZEND_ASSERT(0);
return 0;
} else if (!(ins & 0xf800)) { /* B, BL */
if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) {
ZEND_ASSERT(0);
return 0;
}
} else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */
if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) {
ZEND_ASSERT(0);
return 0;
}
} else if ((ins & 0x3000) == 0x2000) { /* ADR */
ZEND_ASSERT(0);
return 0;
} else if ((ins & 0x3000) == 0x3000) { /* ADRP */
ZEND_ASSERT(0);
return 0;
} else if ((ins & 0x1000)) { /* TBZ, TBNZ */
if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) {
ZEND_ASSERT(0);
return 0;
}
} else {
ZEND_ASSERT(0);
return 0;
}
return n;
}
}
}
veneer = (char*)buffer + (Dst->codesize + dasm_venners_size);
if (veneer > dasm_end) {
return 0; /* jit_buffer_size overflow */
}
na = (ptrdiff_t)veneer - (ptrdiff_t)cp + 4;
n = (int)na;
/* check if we can jump to veneer */
if ((ptrdiff_t)n != na) {
ZEND_ASSERT(0);
return 0;
} else if (!(ins & 0xf800)) { /* B, BL */
if ((n & 3) != 0 || ((n+0x08000000) >> 28) != 0) {
ZEND_ASSERT(0);
return 0;
}
} else if ((ins & 0x800)) { /* B.cond, CBZ, CBNZ, LDR* literal */
if ((n & 3) != 0 || ((n+0x00100000) >> 21) != 0) {
ZEND_ASSERT(0);
return 0;
}
} else if ((ins & 0x3000) == 0x2000) { /* ADR */
ZEND_ASSERT(0);
return 0;
} else if ((ins & 0x3000) == 0x3000) { /* ADRP */
ZEND_ASSERT(0);
return 0;
} else if ((ins & 0x1000)) { /* TBZ, TBNZ */
if ((n & 3) != 0 || ((n+0x00008000) >> 16) != 0) {
ZEND_ASSERT(0);
return 0;
}
} else if ((ins & 0x8000)) { /* absolute */
ZEND_ASSERT(0);
return 0;
} else {
ZEND_ASSERT(0);
return 0;
}
// TODO: support for long veneers (above 128MB) ???
/* check if we can use B to jump from veneer */
na = (ptrdiff_t)cp + offset - (ptrdiff_t)veneer - 4;
m = (int)na;
if ((ptrdiff_t)m != na) {
ZEND_ASSERT(0);
return 0;
} else if ((m & 3) != 0 || ((m+0x08000000) >> 28) != 0) {
ZEND_ASSERT(0);
return 0;
}
/* generate B instruction */
*(uint32_t*)veneer = 0x14000000 | ((m >> 2) & 0x03ffffff);
dasm_venners_size += 4;
if ((ins >> 16) == DASM_REL_LG
&& *(b-1) < 0) {
/* reuse this veneer for the future jumps to global label */
dasm_labels_veneers[-*(b-1)] = veneer;
/* Dst->globals[*(b-1)] = veneer; */
#ifdef HAVE_DISASM
if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) {
const char *name = zend_jit_disasm_find_symbol((ptrdiff_t)cp + offset - 4, (int64_t *)(&offset));
if (name && !offset) {
if (strstr(name, "@veneer") == NULL) {
char *new_name;
zend_spprintf(&new_name, 0, "%s@veneer", name);
zend_jit_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4);
efree(new_name);
} else {
zend_jit_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4);
}
}
}
#endif
}
return n;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* indent-tabs-mode: t
* End:
*/
Reference in New Issue View Git Blame Copy Permalink