php/archived-php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2026-03-24 00:02:20 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
3abebf3e31bd0b02a82e4c4aaeb40463ab055272
archived-php-src/ext/opcache/jit/zend_jit_x86.dasc
Arnaud Le Blanc 3abebf3e31 Fix JIT TLS on MacOS 
The dynamic loader, starting around version 1284, patches the thunk emitted for
thread local variables by the compiler, so that its format changes from

struct Thunk {
    void *func;
    size_t module;
    size_t offset;
}

to

struct Thunk_v2 {
     void *func;
     uint32_t module;
     uint32_t offset;
     // other fields
}

which has the same size, but not the same layout.

This is mentionned in
9307719dd8/libdyld/ThreadLocalVariables.h (L90)

As a result, access to thread specific variables in JIT is broken.

Fix by using the new layout when the new dynamic loader is in use.

Closes GH-20121
2025-10-13 16:16:39 +02:00

16566 lines
471 KiB
Plaintext
Raw Blame History

/*
* +----------------------------------------------------------------------+
* | Zend JIT |
* +----------------------------------------------------------------------+
* | Copyright (c) The PHP Group |
* +----------------------------------------------------------------------+
* | This source file is subject to version 3.01 of the PHP license, |
* | that is bundled with this package in the file LICENSE, and is |
* | available through the world-wide-web at the following url: |
* | https://www.php.net/license/3_01.txt |
* | If you did not receive a copy of the PHP license and are unable to |
* | obtain it through the world-wide-web, please send a note to |
* | license@php.net so we can mail you a copy immediately. |
* +----------------------------------------------------------------------+
* | Authors: Dmitry Stogov <dmitry@php.net> |
* | Xinchen Hui <laruence@php.net> |
* +----------------------------------------------------------------------+
*/
|.if X64
|.arch x64
|.else
|.arch x86
|.endif
|.if X64WIN
|.define FP, r14
|.define IP, r15
|.define IPl, r15d
|.define RX, r15 // the same as VM IP reused as a general purpose reg
|.define CARG1, rcx // x64/POSIX C call arguments.
|.define CARG2, rdx
|.define CARG3, r8
|.define CARG4, r9
|.define CARG1d, ecx
|.define CARG2d, edx
|.define CARG3d, r8d
|.define CARG4d, r9d
|.define FCARG1a, CARG1 // Simulate x86 fastcall.
|.define FCARG2a, CARG2
|.define FCARG1d, CARG1d
|.define FCARG2d, CARG2d
|.define SPAD, 0x58 // padding for CPU stack alignment
|.define NR_SPAD, 0x58 // padding for CPU stack alignment
|.define T3, [r4+0x50] // Used to store old value of IP
|.define T2, [r4+0x48] // Used to store old value of FP
|.define T1, [r4+0x40]
|.define A6, [r4+0x28] // preallocated slot for 6-th argument
|.define A5, [r4+0x20] // preallocated slot for 5-th argument
|.elif X64
|.define FP, r14
|.define IP, r15
|.define IPl, r15d
|.define RX, r15 // the same as VM IP reused as a general purpose reg
|.define CARG1, rdi // x64/POSIX C call arguments.
|.define CARG2, rsi
|.define CARG3, rdx
|.define CARG4, rcx
|.define CARG5, r8
|.define CARG6, r9
|.define CARG1d, edi
|.define CARG2d, esi
|.define CARG3d, edx
|.define CARG4d, ecx
|.define CARG5d, r8d
|.define CARG6d, r9d
|.define FCARG1a, CARG1 // Simulate x86 fastcall.
|.define FCARG2a, CARG2
|.define FCARG1d, CARG1d
|.define FCARG2d, CARG2d
|.define SPAD, 0x18 // padding for CPU stack alignment
|.define NR_SPAD, 0x28 // padding for CPU stack alignment
|.define T3, [r4+0x20] // Used to store old value of IP (CALL VM only)
|.define T2, [r4+0x18] // Used to store old value of FP (CALL VM only)
|.define T1, [r4]
|.else
|.define FP, esi
|.define IP, edi
|.define IPl, edi
|.define RX, edi // the same as VM IP reused as a general purpose reg
|.define FCARG1a, ecx // x86 fastcall arguments.
|.define FCARG2a, edx
|.define FCARG1d, ecx
|.define FCARG2d, edx
|.define SPAD, 0x1c // padding for CPU stack alignment
|.define NR_SPAD, 0x1c // padding for CPU stack alignment
|.define T3, [r4+0x18] // Used to store old value of IP (CALL VM only)
|.define T2, [r4+0x14] // Used to store old value of FP (CALL VM only)
|.define T1, [r4]
|.define A4, [r4+0xC] // preallocated slots for arguments of "cdecl" functions (intersect with T1)
|.define A3, [r4+0x8]
|.define A2, [r4+0x4]
|.define A1, [r4]
|.endif
|.define HYBRID_SPAD, 16 // padding for stack alignment
#ifdef _WIN64
# define TMP_ZVAL_OFFSET 0x20
#else
# define TMP_ZVAL_OFFSET 0
#endif
#define DASM_ALIGNMENT 16
/* According to x86 and x86_64 ABI, CPU stack has to be 16 byte aligned to
* guarantee proper alignment of 128-bit SSE data allocated on stack.
* With broken alignment any execution of SSE code, including calls to
* memcpy() and others, may lead to crash.
*/
const char* zend_reg_name[] = {
#if defined(__x86_64__) || defined(_M_X64)
"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
"r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15",
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
"xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"
#else
"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
"xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"
#endif
};
/* Simulate x86 fastcall */
#ifdef _WIN64
# define ZREG_FCARG1 ZREG_RCX
# define ZREG_FCARG2 ZREG_RDX
#elif defined(__x86_64__)
# define ZREG_FCARG1 ZREG_RDI
# define ZREG_FCARG2 ZREG_RSI
#else
# define ZREG_FCARG1 ZREG_RCX
# define ZREG_FCARG2 ZREG_RDX
#endif
|.type EX, zend_execute_data, FP
|.type OP, zend_op
|.type ZVAL, zval
|.actionlist dasm_actions
|.globals zend_lb
|.section code, cold_code, jmp_table
static void* dasm_labels[zend_lb_MAX];
#if ZTS
static size_t tsrm_ls_cache_tcb_offset = 0;
static size_t tsrm_tls_index;
static size_t tsrm_tls_offset;
#endif
#define IS_32BIT(addr) (((uintptr_t)(addr)) <= 0x7fffffff)
#define IS_SIGNED_32BIT(val) ((((intptr_t)(val)) <= 0x7fffffff) && (((intptr_t)(val)) >= (-2147483647 - 1)))
/* Call range is before or after 2GB */
#define MAY_USE_32BIT_ADDR(addr) \
(IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_buf) && \
IS_SIGNED_32BIT((char*)(addr) - (char*)dasm_end))
#define CAN_USE_AVX() (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX)
/* Not Implemented Yet */
|.macro NIY
|| //ZEND_ASSERT(0);
| int3
|.endmacro
|.macro NIY_STUB
|| //ZEND_ASSERT(0);
| int3
|.endmacro
|.macro ADD_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| add r4, HYBRID_SPAD
||#endif
|.endmacro
|.macro SUB_HYBRID_SPAD
||#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| sub r4, HYBRID_SPAD
||#endif
|.endmacro
|.macro LOAD_ADDR, reg, addr
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mov reg, ((ptrdiff_t)addr) // 0x48 0xc7 0xc0 <imm-32-bit>
|| } else {
| mov64 reg, ((ptrdiff_t)addr) // 0x48 0xb8 <imm-64-bit>
|| }
| .else
| mov reg, ((ptrdiff_t)addr)
| .endif
|.endmacro
|.macro LOAD_TSRM_CACHE, reg
| .if X64WIN
| gs
| mov reg, aword [0x58]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
| .elif WIN
| fs
| mov reg, aword [0x2c]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
| .elif X64APPLE
| gs
|| if (tsrm_ls_cache_tcb_offset) {
| mov reg, aword [tsrm_ls_cache_tcb_offset]
|| } else {
| mov reg, aword [tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
|| }
| .elif X64
| fs
|| if (tsrm_ls_cache_tcb_offset) {
| mov reg, aword [tsrm_ls_cache_tcb_offset]
|| } else {
| mov reg, [0x8]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
|| }
| .else
| gs
|| if (tsrm_ls_cache_tcb_offset) {
| mov reg, aword [tsrm_ls_cache_tcb_offset]
|| } else {
| mov reg, [0x4]
| mov reg, aword [reg+tsrm_tls_index]
| mov reg, aword [reg+tsrm_tls_offset]
|| }
| .endif
|.endmacro
|.macro LOAD_ADDR_ZTS, reg, struct, field
| .if ZTS
| LOAD_TSRM_CACHE reg
| lea reg, aword [reg + (struct.._offset + offsetof(zend_..struct, field))]
| .else
| LOAD_ADDR reg, &struct.field
| .endif
|.endmacro
|.macro PUSH_ADDR, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| push ((ptrdiff_t)addr)
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| push tmp_reg
|| }
| .else
| push ((ptrdiff_t)addr)
| .endif
|.endmacro
|.macro ADDR_STORE, mem, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mov mem, ((ptrdiff_t)addr)
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| mov mem, tmp_reg
|| }
| .else
| mov mem, ((ptrdiff_t)addr)
| .endif
|.endmacro
|.macro ADDR_CMP, mem, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| cmp mem, ((ptrdiff_t)addr)
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| cmp mem, tmp_reg
|| }
| .else
| cmp mem, ((ptrdiff_t)addr)
| .endif
|.endmacro
|.macro PUSH_ADDR_ZTS, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE tmp_reg
| lea tmp_reg, aword [tmp_reg + (struct.._offset + offsetof(zend_..struct, field))]
| push tmp_reg
| .else
| PUSH_ADDR &struct.field, tmp_reg
| .endif
|.endmacro
|.macro _MEM_OP, mem_ins, prefix, addr, op2, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mem_ins prefix [addr], op2
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| mem_ins prefix [tmp_reg], op2
|| }
| .else
| mem_ins prefix [addr], op2
| .endif
|.endmacro
|.macro MEM_LOAD_OP, mem_ins, reg, prefix, addr, tmp_reg
| .if X64
|| if (IS_SIGNED_32BIT(addr)) {
| mem_ins reg, prefix [addr]
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)addr)
| mem_ins reg, prefix [tmp_reg]
|| }
| .else
| mem_ins reg, prefix [addr]
| .endif
|.endmacro
|.macro MEM_LOAD, op1, prefix, addr, tmp_reg
| MEM_LOAD_OP mov, op1, prefix, addr, tmp_reg
|.endmacro
|.macro _MEM_OP_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE tmp_reg
| mem_ins prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))], op2
| .else
| _MEM_OP mem_ins, prefix, &struct.field, op2, tmp_reg
| .endif
|.endmacro
|.macro MEM_STORE_ZTS, prefix, struct, field, op2, tmp_reg
| _MEM_OP_ZTS mov, prefix, struct, field, op2, tmp_reg
|.endmacro
|.macro MEM_CMP_ZTS, prefix, struct, field, op2, tmp_reg
| _MEM_OP_ZTS cmp, prefix, struct, field, op2, tmp_reg
|.endmacro
|.macro MEM_UPDATE_ZTS, mem_ins, prefix, struct, field, op2, tmp_reg
| _MEM_OP_ZTS mem_ins, prefix, struct, field, op2, tmp_reg
|.endmacro
|.macro MEM_LOAD_OP_ZTS, mem_ins, reg, prefix, struct, field, tmp_reg
| .if ZTS
| LOAD_TSRM_CACHE tmp_reg
| mem_ins reg, prefix [tmp_reg+(struct.._offset+offsetof(zend_..struct, field))]
| .else
| MEM_LOAD_OP mem_ins, reg, prefix, &struct.field, tmp_reg
| .endif
|.endmacro
|.macro MEM_LOAD_ZTS, reg, prefix, struct, field, tmp_reg
| MEM_LOAD_OP_ZTS mov, reg, prefix, struct, field, tmp_reg
|.endmacro
|.macro EXT_CALL, func, tmp_reg
| .if X64
|| if (MAY_USE_32BIT_ADDR(func)) {
| call qword &func
|| } else {
| LOAD_ADDR tmp_reg, func
| call tmp_reg
|| }
| .else
| call dword &func
| .endif
|.endmacro
|.macro EXT_JMP, func, tmp_reg
| .if X64
|| if (MAY_USE_32BIT_ADDR(func)) {
| jmp qword &func
|| } else {
| LOAD_ADDR tmp_reg, func
| jmp tmp_reg
|| }
| .else
| jmp dword &func
| .endif
|.endmacro
|.macro SAVE_IP
|| if (GCC_GLOBAL_REGS) {
| mov aword EX->opline, IP
|| }
|.endmacro
|.macro LOAD_IP
|| if (GCC_GLOBAL_REGS) {
| mov IP, aword EX->opline
|| }
|.endmacro
|.macro LOAD_IP_ADDR, addr
|| if (GCC_GLOBAL_REGS) {
| LOAD_ADDR IP, addr
|| } else {
| ADDR_STORE aword EX->opline, addr, RX
|| }
|.endmacro
|.macro LOAD_IP_ADDR_ZTS, struct, field
| .if ZTS
|| if (GCC_GLOBAL_REGS) {
| LOAD_TSRM_CACHE IP
| lea IP, aword [IP + (struct.._offset + offsetof(zend_..struct, field))]
|| } else {
| LOAD_TSRM_CACHE RX
| lea RX, aword [RX + (struct.._offset + offsetof(zend_..struct, field))]
| mov aword EX->opline, RX
|| }
| .else
| LOAD_IP_ADDR &struct.field
| .endif
|.endmacro
|.macro GET_IP, reg
|| if (GCC_GLOBAL_REGS) {
| mov reg, IP
|| } else {
| mov reg, aword EX->opline
|| }
|.endmacro
|.macro ADD_IP, val
|| if (GCC_GLOBAL_REGS) {
| add IP, val
|| } else {
| add aword EX->opline, val
|| }
|.endmacro
|.macro JMP_IP
|| if (GCC_GLOBAL_REGS) {
| jmp aword [IP]
|| } else {
| mov r0, aword EX:FCARG1a->opline
| jmp aword [r0]
|| }
|.endmacro
/* In 64-bit build we compare only low 32-bits.
* x86_64 cmp instruction doesn't support immediate 64-bit operand, and full
* comparison would require an additional load of 64-bit address into register.
* This is not a problem at all, while JIT buffer size is less than 4GB.
*/
|.macro CMP_IP, addr
|| if (GCC_GLOBAL_REGS) {
| cmp IPl, addr
|| } else {
| cmp dword EX->opline, addr
|| }
|.endmacro
|.macro LOAD_ZVAL_ADDR, reg, addr
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| LOAD_ADDR reg, Z_ZV(addr)
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|| if (Z_OFFSET(addr)) {
| lea reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else {
| mov reg, Ra(Z_REG(addr))
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro PUSH_ZVAL_ADDR, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| PUSH_ADDR Z_ZV(addr), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|| if (Z_OFFSET(addr)) {
| lea tmp_reg, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
| push tmp_reg
|| } else {
| push Ra(Z_REG(addr))
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro GET_Z_TYPE_INFO, reg, zv
| mov reg, dword [zv+offsetof(zval,u1.type_info)]
|.endmacro
|.macro SET_Z_TYPE_INFO, zv, type
| mov dword [zv+offsetof(zval,u1.type_info)], type
|.endmacro
|.macro GET_ZVAL_TYPE, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.v.type)]
|.endmacro
|.macro GET_ZVAL_TYPE_INFO, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)]
|.endmacro
|.macro SET_ZVAL_TYPE_INFO, addr, type
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval,u1.type_info)], type
|.endmacro
|.macro GET_Z_PTR, reg, zv
| mov reg, aword [zv]
|.endmacro
|.macro GET_Z_W2, reg, zv
| mov reg, dword [zv+4]
|.endmacro
|.macro SET_Z_W2, zv, reg
| mov dword [zv+4], reg
|.endmacro
|.macro GET_ZVAL_PTR, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|.endmacro
|.macro SET_ZVAL_PTR, addr, val
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], val
|.endmacro
|.macro GET_ZVAL_W2, reg, addr
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov reg, dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4]
|.endmacro
|.macro SET_ZVAL_W2, addr, val
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov dword [Ra(Z_REG(addr))+Z_OFFSET(addr)+4], val
|.endmacro
|.macro UNDEF_OPLINE_RESULT
| mov r0, EX->opline
| mov eax, dword OP:r0->result.var
| SET_Z_TYPE_INFO FP + r0, IS_UNDEF
|.endmacro
|.macro UNDEF_OPLINE_RESULT_IF_USED
| test byte OP:RX->result_type, (IS_TMP_VAR|IS_VAR)
| jz >1
| mov eax, dword OP:RX->result.var
| SET_Z_TYPE_INFO FP + r0, IS_UNDEF
|1:
|.endmacro
|.macro SSE_AVX_INS, sse_ins, avx_ins, op1, op2
|| if (CAN_USE_AVX()) {
| avx_ins op1, op2
|| } else {
| sse_ins op1, op2
|| }
|.endmacro
|.macro SSE_OP, sse_ins, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| MEM_LOAD_OP sse_ins, xmm(reg-ZREG_XMM0), qword, Z_ZV(addr), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| sse_ins xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| sse_ins xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro DOUBLE_CMP, reg, addr
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| .if X64
|| if (IS_SIGNED_32BIT(Z_ZV(addr))) {
| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
|| } else {
| LOAD_ADDR r0, Z_ZV(addr)
| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [r0]
|| }
| .else
| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
| .endif
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| SSE_AVX_INS ucomisd, vucomisd, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro DOUBLE_GET_LONG, reg, lval, tmp_reg
|| if (lval == 0) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
|| }
|| } else {
|.if X64
|| if (!IS_SIGNED_32BIT(lval)) {
| mov64 Ra(tmp_reg), lval
|| } else {
| mov Ra(tmp_reg), lval
|| }
|.else
| mov Ra(tmp_reg), lval
|.endif
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| vcvtsi2sd, xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(tmp_reg)
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| cvtsi2sd, xmm(reg-ZREG_XMM0), Ra(tmp_reg)
|| }
|| }
|.endmacro
|.macro DOUBLE_GET_ZVAL_LVAL, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| DOUBLE_GET_LONG reg, Z_LVAL_P(Z_ZV(addr)), tmp_reg
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| cvtsi2sd xmm(reg-ZREG_XMM0), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| }
|| } else if (Z_MODE(addr) == IS_REG) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| vcvtsi2sd xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
|| } else {
| xorps xmm(reg-ZREG_XMM0), xmm(reg-ZREG_XMM0)
| cvtsi2sd xmm(reg-ZREG_XMM0), Ra(Z_REG(addr))
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro DOUBLE_GET_ZVAL_DVAL, reg, addr
|| if (Z_MODE(addr) != IS_REG || reg != Z_REG(addr)) {
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| .if X64
|| if (IS_SIGNED_32BIT(Z_ZV(addr))) {
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
|| } else {
| LOAD_ADDR r0, Z_ZV(addr)
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [r0]
|| }
| .else
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Z_ZV(addr)]
| .endif
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| SSE_AVX_INS movsd, vmovsd, xmm(reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| SSE_AVX_INS movaps, vmovaps, xmm(reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|| }
|.endmacro
|.macro SSE_MATH, opcode, reg, addr, tmp_reg
|| switch (opcode) {
|| case ZEND_ADD:
| SSE_OP addsd, reg, addr, tmp_reg
|| break;
|| case ZEND_SUB:
| SSE_OP subsd, reg, addr, tmp_reg
|| break;
|| case ZEND_MUL:
| SSE_OP mulsd, reg, addr, tmp_reg
|| break;
|| case ZEND_DIV:
| SSE_OP divsd, reg, addr, tmp_reg
|| break;
|| }
|.endmacro
|.macro SSE_MATH_REG, opcode, dst_reg, src_reg
|| switch (opcode) {
|| case ZEND_ADD:
| addsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_SUB:
| subsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_MUL:
| mulsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_DIV:
| divsd xmm(dst_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| }
|.endmacro
|.macro DOUBLE_SET_ZVAL_DVAL, addr, reg
|| if (Z_MODE(addr) == IS_REG) {
|| if (reg != Z_REG(addr)) {
| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(addr)-ZREG_XMM0), xmm(reg-ZREG_XMM0)
|| }
|| } else {
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| SSE_AVX_INS movsd, vmovsd, qword [Ra(Z_REG(addr))+Z_OFFSET(addr)], xmm(reg-ZREG_XMM0)
|| }
|.endmacro
|.macro AVX_OP, avx_ins, reg, op1_reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| .if X64
|| if (IS_SIGNED_32BIT(Z_ZV(addr))) {
| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Z_ZV(addr)]
|| } else {
| mov64 tmp_reg, ((ptrdiff_t)Z_ZV(addr))
| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [tmp_reg]
|| }
| .else
| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [addr]
| .endif
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), qword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| avx_ins xmm(reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(Z_REG(addr)-ZREG_XMM0)
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro AVX_MATH, opcode, reg, op1_reg, addr, tmp_reg
|| switch (opcode) {
|| case ZEND_ADD:
| AVX_OP vaddsd, reg, op1_reg, addr, tmp_reg
|| break;
|| case ZEND_SUB:
| AVX_OP vsubsd, reg, op1_reg, addr, tmp_reg
|| break;
|| case ZEND_MUL:
| AVX_OP vmulsd, reg, op1_reg, addr, tmp_reg
|| break;
|| case ZEND_DIV:
| AVX_OP vdivsd, reg, op1_reg, addr, tmp_reg
|| break;
|| }
|.endmacro
|.macro AVX_MATH_REG, opcode, dst_reg, op1_reg, src_reg
|| switch (opcode) {
|| case ZEND_ADD:
| vaddsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_SUB:
| vsubsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_MUL:
| vmulsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| case ZEND_DIV:
| vdivsd xmm(dst_reg-ZREG_XMM0), xmm(op1_reg-ZREG_XMM0), xmm(src_reg-ZREG_XMM0)
|| break;
|| }
|.endmacro
|.macro LONG_OP, long_ins, reg, addr, tmp_reg
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) {
| mov64 tmp_reg, Z_LVAL_P(Z_ZV(addr))
| long_ins Ra(reg), tmp_reg
|| } else {
| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr))
|| }
| .else
| long_ins Ra(reg), Z_LVAL_P(Z_ZV(addr))
| .endif
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| long_ins Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
| long_ins Ra(reg), Ra(Z_REG(addr))
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_OP_WITH_32BIT_CONST, long_ins, op1_addr, lval
|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) {
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
|| } else if (Z_MODE(op1_addr) == IS_REG) {
| long_ins Ra(Z_REG(op1_addr)), lval
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_OP_WITH_CONST, long_ins, op1_addr, lval
|| if (Z_MODE(op1_addr) == IS_MEM_ZVAL) {
| .if X64
|| if (!IS_SIGNED_32BIT(lval)) {
| mov64 r0, lval
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], r0
|| } else {
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
|| }
| .else
| long_ins aword [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)], lval
| .endif
|| } else if (Z_MODE(op1_addr) == IS_REG) {
| .if X64
|| if (!IS_SIGNED_32BIT(lval)) {
| mov64 r0, lval
| long_ins Ra(Z_REG(op1_addr)), r0
|| } else {
| long_ins Ra(Z_REG(op1_addr)), lval
|| }
| .else
| long_ins Ra(Z_REG(op1_addr)), lval
| .endif
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro GET_ZVAL_LVAL, reg, addr
|| if (Z_MODE(addr) == IS_CONST_ZVAL) {
|| if (Z_LVAL_P(Z_ZV(addr)) == 0) {
| xor Ra(reg), Ra(reg)
|| } else {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(addr)))) {
| mov64 Ra(reg), Z_LVAL_P(Z_ZV(addr))
|| } else {
| mov Ra(reg), Z_LVAL_P(Z_ZV(addr))
|| }
| .else
| mov Ra(reg), Z_LVAL_P(Z_ZV(addr))
| .endif
|| }
|| } else if (Z_MODE(addr) == IS_MEM_ZVAL) {
| mov Ra(reg), aword [Ra(Z_REG(addr))+Z_OFFSET(addr)]
|| } else if (Z_MODE(addr) == IS_REG) {
|| if (reg != Z_REG(addr)) {
| mov Ra(reg), Ra(Z_REG(addr))
|| }
|| } else {
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_MATH, opcode, reg, addr, tmp_reg
|| switch (opcode) {
|| case ZEND_ADD:
| LONG_OP add, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_SUB:
| LONG_OP sub, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_MUL:
| LONG_OP imul, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_BW_OR:
| LONG_OP or, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_BW_AND:
| LONG_OP and, reg, addr, Ra(tmp_reg)
|| break;
|| case ZEND_BW_XOR:
| LONG_OP xor, reg, addr, Ra(tmp_reg)
|| break;
|| default:
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro LONG_MATH_REG, opcode, dst_reg, src_reg
|| switch (opcode) {
|| case ZEND_ADD:
| add dst_reg, src_reg
|| break;
|| case ZEND_SUB:
| sub dst_reg, src_reg
|| break;
|| case ZEND_MUL:
| imul dst_reg, src_reg
|| break;
|| case ZEND_BW_OR:
| or dst_reg, src_reg
|| break;
|| case ZEND_BW_AND:
| and dst_reg, src_reg
|| break;
|| case ZEND_BW_XOR:
| xor dst_reg, src_reg
|| break;
|| default:
|| ZEND_UNREACHABLE();
|| }
|.endmacro
|.macro SET_ZVAL_LVAL, addr, lval
|| if (Z_MODE(addr) == IS_REG) {
| mov Ra(Z_REG(addr)), lval
|| } else {
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| mov aword [Ra(Z_REG(addr))+Z_OFFSET(addr)], lval
|| }
|.endmacro
|.macro ZVAL_COPY_CONST, dst_addr, dst_info, dst_def_info, zv, tmp_reg
|| if (Z_TYPE_P(zv) > IS_TRUE) {
|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0;
|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| } else {
| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| }
| .if X64
|| } else if (!IS_SIGNED_32BIT(zv)) {
| mov64 Ra(tmp_reg), ((uintptr_t)zv)
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)]
| .endif
|| } else {
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)]
|| }
| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg
|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ? Z_REG(dst_addr) : ZREG_XMM0;
| DOUBLE_GET_LONG dst_reg, Z_LVAL_P(zv), ZREG_R0
| DOUBLE_SET_ZVAL_DVAL dst_addr, dst_reg
|| } else if (Z_LVAL_P(zv) == 0 && Z_MODE(dst_addr) == IS_REG) {
| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr))
|| } else {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv))
|| } else {
| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg)
|| }
|| } else {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
|| }
| .else
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| .endif
|| }
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE
|| }
|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|.endmacro
|.macro ZVAL_COPY_CONST_2, dst_addr, res_addr, dst_info, dst_def_info, zv, tmp_reg
|| if (Z_TYPE_P(zv) > IS_TRUE) {
|| if (Z_TYPE_P(zv) == IS_DOUBLE) {
|| zend_reg dst_reg = (Z_MODE(dst_addr) == IS_REG) ?
|| Z_REG(dst_addr) : ((Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0);
|| if (Z_DVAL_P(zv) == 0.0 && !is_signed(Z_DVAL_P(zv))) {
|| if (CAN_USE_AVX()) {
| vxorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| } else {
| xorps xmm(dst_reg-ZREG_XMM0), xmm(dst_reg-ZREG_XMM0)
|| }
| .if X64
|| } else if (!IS_SIGNED_32BIT(zv)) {
| mov64 Ra(tmp_reg), ((uintptr_t)zv)
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [Ra(tmp_reg)]
| .endif
|| } else {
| SSE_AVX_INS movsd, vmovsd, xmm(dst_reg-ZREG_XMM0), qword [((uint32_t)(uintptr_t)zv)]
|| }
| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
|| } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| DOUBLE_GET_LONG Z_REG(dst_addr), Z_LVAL_P(zv), ZREG_R0
| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr)
|| } else if (Z_MODE(res_addr) == IS_REG) {
| DOUBLE_GET_LONG Z_REG(res_addr), Z_LVAL_P(zv), ZREG_R0
| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr)
|| } else {
| DOUBLE_GET_LONG ZREG_XMM0, Z_LVAL_P(zv), ZREG_R0
| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
|| }
|| } else if (Z_LVAL_P(zv) == 0 && (Z_MODE(dst_addr) == IS_REG || Z_MODE(res_addr) == IS_REG)) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| xor Ra(Z_REG(dst_addr)), Ra(Z_REG(dst_addr))
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else {
| xor Ra(Z_REG(res_addr)), Ra(Z_REG(res_addr))
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| }
|| } else {
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
|| if (Z_MODE(dst_addr) == IS_REG) {
| mov64 Ra(Z_REG(dst_addr)), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else if (Z_MODE(res_addr) == IS_REG) {
| mov64 Ra(Z_REG(res_addr)), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| mov64 Ra(tmp_reg), ((uintptr_t)Z_LVAL_P(zv))
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg)
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg)
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else if (Z_MODE(res_addr) == IS_REG) {
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
|| }
| .else
|| if (Z_MODE(dst_addr) == IS_REG) {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| } else if (Z_MODE(res_addr) == IS_REG) {
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| SET_ZVAL_LVAL dst_addr, Z_LVAL_P(zv)
| SET_ZVAL_LVAL res_addr, Z_LVAL_P(zv)
|| }
| .endif
|| }
|| }
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO dst_addr, IS_DOUBLE
|| }
|| } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<<Z_TYPE_P(zv))) || (dst_info & (MAY_BE_STRING|MAY_BE_ARRAY)) != 0) {
| SET_ZVAL_TYPE_INFO dst_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
|| if (dst_def_info == MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
|| } else {
| SET_ZVAL_TYPE_INFO res_addr, Z_TYPE_INFO_P(zv)
|| }
|| }
|.endmacro
/* the same as above, but "src" may overlap with "tmp_reg1" */
|.macro ZVAL_COPY_VALUE, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
| ZVAL_COPY_VALUE_V dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
|| !(src_info & MAY_BE_GUARD) &&
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD))) {
|| uint8_t type = concrete_type(src_info);
| SET_ZVAL_TYPE_INFO dst_addr, type
|| }
|| }
|| } else {
| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr
| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1)
|| }
|.endmacro
|.macro ZVAL_COPY_VALUE_V, dst_addr, dst_info, src_addr, src_info, tmp_reg1, tmp_reg2
|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
|| if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_LONG) {
|| if (Z_MODE(src_addr) == IS_REG) {
|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr))
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr
|| } else {
| GET_ZVAL_LVAL tmp_reg2, src_addr
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2)
|| }
|| } else if ((src_info & (MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
|| if (Z_MODE(src_addr) == IS_REG) {
| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr)
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr
|| } else {
| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr
| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
|| }
|| } else if (!(src_info & (MAY_BE_DOUBLE|MAY_BE_GUARD))) {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
|| } else {
| .if X64
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| .else
|| if ((tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr))) {
| GET_ZVAL_W2 Ra(tmp_reg2), src_addr
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2)
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
|| } else {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| GET_ZVAL_W2 Ra(tmp_reg1), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1)
|| }
| .endif
|| }
|| }
|.endmacro
|.macro ZVAL_COPY_VALUE_2, dst_addr, dst_info, res_addr, src_addr, src_info, tmp_reg1, tmp_reg2
|| if (src_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
|| if ((src_info & MAY_BE_ANY) == MAY_BE_LONG) {
|| if (Z_MODE(src_addr) == IS_REG) {
|| if (Z_MODE(dst_addr) != IS_REG || Z_REG(dst_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(src_addr))
|| }
|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(src_addr)) {
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(src_addr))
|| }
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(dst_addr), src_addr
|| if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != Z_REG(dst_addr)) {
| SET_ZVAL_LVAL res_addr, Ra(Z_REG(dst_addr))
|| }
|| } else if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_LVAL Z_REG(res_addr), src_addr
| SET_ZVAL_LVAL dst_addr, Ra(Z_REG(res_addr))
|| } else {
| GET_ZVAL_LVAL tmp_reg2, src_addr
| SET_ZVAL_LVAL dst_addr, Ra(tmp_reg2)
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg2)
|| }
|| } else if ((src_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
|| if (Z_MODE(src_addr) == IS_REG) {
| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(src_addr)
| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(src_addr)
|| } else if (Z_MODE(dst_addr) == IS_REG) {
| DOUBLE_GET_ZVAL_DVAL Z_REG(dst_addr), src_addr
| DOUBLE_SET_ZVAL_DVAL res_addr, Z_REG(dst_addr)
|| } else if (Z_MODE(res_addr) == IS_REG) {
| DOUBLE_GET_ZVAL_DVAL Z_REG(res_addr), src_addr
| DOUBLE_SET_ZVAL_DVAL dst_addr, Z_REG(res_addr)
|| } else {
| DOUBLE_GET_ZVAL_DVAL ZREG_XMM0, src_addr
| DOUBLE_SET_ZVAL_DVAL dst_addr, ZREG_XMM0
| DOUBLE_SET_ZVAL_DVAL res_addr, ZREG_XMM0
|| }
|| } else if (!(src_info & MAY_BE_DOUBLE)) {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
|| } else {
| .if X64
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
| .else
|| if (tmp_reg1 == tmp_reg2 || tmp_reg1 == Z_REG(src_addr)) {
| GET_ZVAL_W2 Ra(tmp_reg2), src_addr
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg2)
| SET_ZVAL_W2 res_addr, Ra(tmp_reg2)
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
|| } else {
| GET_ZVAL_PTR Ra(tmp_reg2), src_addr
| GET_ZVAL_W2 Ra(tmp_reg1), src_addr
| SET_ZVAL_PTR dst_addr, Ra(tmp_reg2)
| SET_ZVAL_PTR res_addr, Ra(tmp_reg2)
| SET_ZVAL_W2 dst_addr, Ra(tmp_reg1)
| SET_ZVAL_W2 res_addr, Ra(tmp_reg1)
|| }
| .endif
|| }
|| }
|| if ((src_info & (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)) &&
|| has_concrete_type(src_info & MAY_BE_ANY)) {
|| uint8_t type = concrete_type(src_info);
|| if (Z_MODE(dst_addr) == IS_MEM_ZVAL) {
|| if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (src_info & (MAY_BE_ANY|MAY_BE_UNDEF))) {
| SET_ZVAL_TYPE_INFO dst_addr, type
|| }
|| }
|| if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, type
|| }
|| } else {
| GET_ZVAL_TYPE_INFO Rd(tmp_reg1), src_addr
| SET_ZVAL_TYPE_INFO dst_addr, Rd(tmp_reg1)
| SET_ZVAL_TYPE_INFO res_addr, Rd(tmp_reg1)
|| }
|.endmacro
|.macro IF_UNDEF, type_reg, label
| test type_reg, type_reg
| je label
|.endmacro
|.macro IF_TYPE, type, val, label
| cmp type, val
| je label
|.endmacro
|.macro IF_NOT_TYPE, type, val, label
| cmp type, val
| jne label
|.endmacro
|.macro IF_Z_TYPE, zv, val, label
| IF_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro IF_NOT_Z_TYPE, zv, val, label
| IF_NOT_TYPE byte [zv+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro CMP_ZVAL_TYPE, addr, val
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| cmp byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val
|.endmacro
|.macro IF_ZVAL_TYPE, addr, val, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro IF_NOT_ZVAL_TYPE, addr, val, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_NOT_TYPE byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type)], val, label
|.endmacro
|.macro IF_FLAGS, type_flags, mask, label
| test type_flags, mask
| jnz label
|.endmacro
|.macro IF_NOT_FLAGS, type_flags, mask, label
| test type_flags, mask
| jz label
|.endmacro
|.macro IF_REFCOUNTED, type_flags, label
| IF_FLAGS type_flags, IS_TYPE_REFCOUNTED, label
|.endmacro
|.macro IF_NOT_REFCOUNTED, type_flags, label
| //IF_NOT_FLAGS type_flags, IS_TYPE_REFCOUNTED, label
| test type_flags, type_flags
| jz label
|.endmacro
|.macro IF_ZVAL_FLAGS, addr, mask, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label
|.endmacro
|.macro IF_NOT_ZVAL_FLAGS, addr, mask, label
|| ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL);
| IF_NOT_FLAGS byte [Ra(Z_REG(addr))+Z_OFFSET(addr)+offsetof(zval, u1.v.type_flags)], mask, label
|.endmacro
|.macro IF_ZVAL_REFCOUNTED, addr, label
| IF_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label
|.endmacro
|.macro IF_NOT_ZVAL_REFCOUNTED, addr, label
| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_REFCOUNTED, label
|.endmacro
|.macro IF_NOT_ZVAL_COLLECTABLE, addr, label
| IF_NOT_ZVAL_FLAGS addr, IS_TYPE_COLLECTABLE, label
|.endmacro
|.macro GC_ADDREF, zv
| add dword [zv], 1
|.endmacro
|.macro GC_DELREF, zv
| sub dword [zv], 1
|.endmacro
|.macro IF_GC_MAY_NOT_LEAK, ptr, label
| test dword [ptr+4],(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT))
| jne label
|.endmacro
|.macro ADDREF_CONST, zv, tmp_reg
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
| add dword [tmp_reg], 1
|| } else {
| add dword [Z_LVAL_P(zv)], 1
|| }
| .else
| add dword [Z_LVAL_P(zv)], 1
| .endif
|.endmacro
|.macro ADDREF_CONST_2, zv, tmp_reg
| .if X64
|| if (!IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
| mov64 tmp_reg, ((uintptr_t)Z_LVAL_P(zv))
| add dword [tmp_reg], 2
|| } else {
| add dword [Z_LVAL_P(zv)], 2
|| }
| .else
| add dword [Z_LVAL_P(zv)], 2
| .endif
|.endmacro
|.macro TRY_ADDREF, val_info, type_flags_reg, value_ptr_reg
|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_REFCOUNTED type_flags_reg, >1
|| }
| GC_ADDREF value_ptr_reg
|1:
|| }
|.endmacro
|.macro TRY_ADDREF_2, val_info, type_flags_reg, value_ptr_reg
|| if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
|| if (val_info & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_REFCOUNTED type_flags_reg, >1
|| }
| add dword [value_ptr_reg], 2
|1:
|| }
|.endmacro
|.macro ZVAL_DEREF, reg, info
|| if (info & MAY_BE_REF) {
| IF_NOT_Z_TYPE, reg, IS_REFERENCE, >1
| GET_Z_PTR reg, reg
| add reg, offsetof(zend_reference, val)
|1:
|| }
|.endmacro
|.macro SET_EX_OPLINE, op, tmp_reg
|| if (op == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| SAVE_IP
|| } else {
| ADDR_STORE aword EX->opline, op, tmp_reg
|| if (!GCC_GLOBAL_REGS) {
|| zend_jit_reset_last_valid_opline();
|| }
|| }
|.endmacro
// zval should be in FCARG1a
|.macro ZVAL_DTOR_FUNC, var_info, opline // arg1 must be in FCARG1a
|| do {
|| if (!((var_info) & MAY_BE_GUARD)
|| && has_concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|| uint8_t type = concrete_type((var_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE));
|| if (type == IS_STRING && !ZEND_DEBUG) {
| EXT_CALL _efree, r0
|| break;
|| } else if (type == IS_ARRAY) {
|| if ((var_info) & (MAY_BE_ARRAY_KEY_STRING|MAY_BE_ARRAY_OF_STRING|MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF)) {
|| if (opline && ((var_info) & (MAY_BE_ARRAY_OF_ARRAY|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_REF))) {
| SET_EX_OPLINE opline, r0
|| }
| EXT_CALL zend_array_destroy, r0
|| } else {
| EXT_CALL zend_jit_array_free, r0
|| }
|| break;
|| } else if (type == IS_OBJECT) {
|| if (opline) {
| SET_EX_OPLINE opline, r0
|| }
| EXT_CALL zend_objects_store_del, r0
|| break;
|| }
|| }
|| if (opline) {
| SET_EX_OPLINE opline, r0
|| }
| EXT_CALL rc_dtor_func, r0
|| } while(0);
|.endmacro
|.macro ZVAL_PTR_DTOR, addr, op_info, gc, cold, opline
|| if ((op_info) & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF|MAY_BE_GUARD)) {
|| if ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
|| if (cold) {
| IF_ZVAL_REFCOUNTED addr, >1
|.cold_code
|1:
|| } else {
| IF_NOT_ZVAL_REFCOUNTED addr, >4
|| }
|| }
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1a, addr
| GC_DELREF FCARG1a
|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_1(op_info)) {
|| if (((op_info) & MAY_BE_GUARD) || RC_MAY_BE_N(op_info)) {
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
| jnz >3
|| } else {
| jnz >4
|| }
|| }
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op_info, opline
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
| jmp >4
|| }
|3:
|| }
|| if (gc && (((op_info) & MAY_BE_GUARD) || (RC_MAY_BE_N(op_info) && ((op_info) & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0))) {
|| if ((op_info) & (MAY_BE_REF|MAY_BE_GUARD)) {
|| zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val));
| IF_NOT_ZVAL_TYPE addr, IS_REFERENCE, >1
| IF_NOT_ZVAL_COLLECTABLE ref_addr, >4
| GET_ZVAL_PTR FCARG1a, ref_addr
|1:
|| }
| IF_GC_MAY_NOT_LEAK FCARG1a, >4
|| if (opline) {
| SET_EX_OPLINE opline, r0
|| }
| // gc_possible_root(Z_COUNTED_P(z))
| EXT_CALL gc_possible_root, r0
|| }
|| if (cold && ((op_info) & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_INDIRECT|MAY_BE_GUARD)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) != 0) {
| jmp >4
|.code
|| }
|4:
|| }
|.endmacro
|.macro FREE_OP, op_type, op, op_info, cold, opline
|| if (op_type & (IS_VAR|IS_TMP_VAR)) {
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var), op_info, 0, cold, opline
|| }
|.endmacro
|.macro SEPARATE_ARRAY, addr, op_info, cold
|| if (RC_MAY_BE_N(op_info)) {
|| if (Z_REG(addr) != ZREG_FP) {
| GET_ZVAL_LVAL ZREG_R0, addr
|| if (RC_MAY_BE_1(op_info)) {
| cmp dword [r0], 1 // if (GC_REFCOUNT() > 1)
| jbe >2
|| }
|| if (Z_REG(addr) != ZREG_FCARG1 || Z_OFFSET(addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, addr
|| }
| EXT_CALL zend_jit_zval_array_dup, r0
|2:
| mov FCARG1a, r0
|| } else {
| GET_ZVAL_LVAL ZREG_FCARG1, addr
|| if (RC_MAY_BE_1(op_info)) {
| cmp dword [FCARG1a], 1 // if (GC_REFCOUNT() > 1)
|| if (cold) {
| ja >1
|.cold_code
|1:
|| } else {
| jbe >2
|| }
|| }
| IF_NOT_ZVAL_REFCOUNTED addr, >1
| GC_DELREF FCARG1a
|1:
| EXT_CALL zend_array_dup, r0
| SET_ZVAL_PTR addr, r0
| SET_ZVAL_TYPE_INFO addr, IS_ARRAY_EX
| mov FCARG1a, r0
|| if (RC_MAY_BE_1(op_info)) {
|| if (cold) {
| jmp >2
|.code
|| }
|| }
|2:
|| }
|| } else {
| GET_ZVAL_LVAL ZREG_FCARG1, addr
|| }
|.endmacro
|.macro EFREE_REG_REFERENCE
||#if ZEND_DEBUG
| xor FCARG2a, FCARG2a // filename
| .if X64WIN
| xor CARG3d, CARG3d // lineno
| xor CARG4, CARG4
| mov aword A5, 0
| EXT_CALL _efree, r0
| .elif X64
| xor CARG3d, CARG3d // lineno
| xor CARG4, CARG4
| xor CARG5, CARG5
| EXT_CALL _efree, r0
| .else
| sub r4, 4
| push 0
| push 0
| push 0 // lineno
| EXT_CALL _efree, r0
| add r4, 4
| .endif
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
| EXT_CALL _efree_32, r0
||#else
| EXT_CALL _efree, r0
||#endif
||#endif
|.endmacro
|.macro EFREE_REFERENCE, ptr
| mov FCARG1a, ptr
| EFREE_REG_REFERENCE
|.endmacro
|.macro EMALLOC, size, op_array, opline
||#if ZEND_DEBUG
|| const char *filename = op_array->filename ? op_array->filename->val : NULL;
| mov FCARG1a, size
| LOAD_ADDR FCARG2a, filename
| .if X64WIN
| mov CARG3d, opline->lineno
| xor CARG4, CARG4
| mov aword A5, 0
| EXT_CALL _emalloc, r0
| .elif X64
| mov CARG3d, opline->lineno
| xor CARG4, CARG4
| xor CARG5, CARG5
| EXT_CALL _emalloc, r0
| .else
| sub r4, 4
| push 0
| push 0
| push opline->lineno
| EXT_CALL _emalloc, r0
| add r4, 4
| .endif
||#else
||#ifdef HAVE_BUILTIN_CONSTANT_P
|| if (size > 24 && size <= 32) {
| EXT_CALL _emalloc_32, r0
|| } else {
| mov FCARG1a, size
| EXT_CALL _emalloc, r0
|| }
||#else
| mov FCARG1a, size
| EXT_CALL _emalloc, r0
||#endif
||#endif
|.endmacro
|.macro OBJ_RELEASE, reg, exit_label
| GC_DELREF Ra(reg)
| jne >1
| // zend_objects_store_del(obj);
|| if (reg != ZREG_FCARG1) {
| mov FCARG1a, Ra(reg)
|| }
| EXT_CALL zend_objects_store_del, r0
| jmp exit_label
|1:
| IF_GC_MAY_NOT_LEAK Ra(reg), >1
| // gc_possible_root(obj)
|| if (reg != ZREG_FCARG1) {
| mov FCARG1a, Ra(reg)
|| }
| EXT_CALL gc_possible_root, r0
|1:
|.endmacro
|.macro UNDEFINED_OFFSET, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| call ->undefined_offset_ex
|| } else {
| SET_EX_OPLINE opline, r0
| call ->undefined_offset
|| }
|.endmacro
|.macro UNDEFINED_INDEX, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| call ->undefined_index_ex
|| } else {
| SET_EX_OPLINE opline, r0
| call ->undefined_index
|| }
|.endmacro
|.macro CANNOT_ADD_ELEMENT, opline
|| if (opline == last_valid_opline) {
|| zend_jit_use_last_valid_opline();
| call ->cannot_add_element_ex
|| } else {
| SET_EX_OPLINE opline, r0
| call ->cannot_add_element
|| }
|.endmacro
|.macro ENDBR
||#if defined (__CET__) && (__CET__ & 1) != 0
| .if X64
| endbr64
| .else
| endbr32
| .endif
||#endif
|.endmacro
#if defined (__CET__) && (__CET__ & 1) != 0
# define ENDBR_PADDING 4
#else
# define ENDBR_PADDING 0
#endif
static bool reuse_ip = 0;
static bool delayed_call_chain = 0;
static uint32_t delayed_call_level = 0;
static const zend_op *last_valid_opline = NULL;
static bool use_last_vald_opline = 0;
static bool track_last_valid_opline = 0;
static int jit_return_label = -1;
static uint32_t current_trace_num = 0;
static uint32_t allowed_opt_flags = 0;
static void zend_jit_track_last_valid_opline(void)
{
use_last_vald_opline = 0;
track_last_valid_opline = 1;
}
static void zend_jit_use_last_valid_opline(void)
{
if (track_last_valid_opline) {
use_last_vald_opline = 1;
track_last_valid_opline = 0;
}
}
static bool zend_jit_trace_uses_initial_ip(void)
{
return use_last_vald_opline;
}
static void zend_jit_set_last_valid_opline(const zend_op *target_opline)
{
if (!reuse_ip) {
track_last_valid_opline = 0;
last_valid_opline = target_opline;
}
}
static void zend_jit_reset_last_valid_opline(void)
{
track_last_valid_opline = 0;
last_valid_opline = NULL;
}
static void zend_jit_start_reuse_ip(void)
{
zend_jit_reset_last_valid_opline();
reuse_ip = 1;
}
static int zend_jit_reuse_ip(dasm_State **Dst)
{
if (!reuse_ip) {
zend_jit_start_reuse_ip();
| // call = EX(call);
| mov RX, EX->call
}
return 1;
}
static void zend_jit_stop_reuse_ip(void)
{
reuse_ip = 0;
}
static int zend_jit_interrupt_handler_stub(dasm_State **Dst)
{
|->interrupt_handler:
| SAVE_IP
| //EG(vm_interrupt) = 0;
| MEM_STORE_ZTS byte, executor_globals, vm_interrupt, 0, r0
| //if (EG(timed_out)) {
| MEM_CMP_ZTS byte, executor_globals, timed_out, 0, r0
| je >1
| //zend_timeout();
| EXT_CALL zend_timeout, r0
|1:
| //} else if (zend_interrupt_function) {
if (zend_interrupt_function) {
| //zend_interrupt_function(execute_data);
|.if X64
| mov CARG1, FP
| EXT_CALL zend_interrupt_function, r0
|.else
| mov aword A1, FP
| EXT_CALL zend_interrupt_function, r0
|.endif
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| je >1
| EXT_CALL zend_jit_exception_in_interrupt_handler_helper, r0
|1:
| //ZEND_VM_ENTER();
| //execute_data = EG(current_execute_data);
| MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0
| LOAD_IP
}
| //ZEND_VM_CONTINUE()
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
return 1;
}
static int zend_jit_exception_handler_stub(dasm_State **Dst)
{
|->exception_handler:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
const void *handler = zend_get_opcode_handler_func(EG(exception_op));
| ADD_HYBRID_SPAD
| EXT_CALL handler, r0
| JMP_IP
} else {
const void *handler = EG(exception_op)->handler;
if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| EXT_JMP handler, r0
} else {
| mov FCARG1a, FP
| EXT_CALL handler, r0
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| test eax, eax
| jl >1
| mov r0, 1 // ZEND_VM_ENTER
|1:
| ret
}
}
return 1;
}
static int zend_jit_exception_handler_undef_stub(dasm_State **Dst)
{
|->exception_handler_undef:
| MEM_LOAD_ZTS r0, aword, executor_globals, opline_before_exception, r0
| test byte OP:r0->result_type, (IS_TMP_VAR|IS_VAR)
| jz >1
| mov eax, dword OP:r0->result.var
| SET_Z_TYPE_INFO FP + r0, IS_UNDEF
|1:
| jmp ->exception_handler
return 1;
}
static int zend_jit_exception_handler_free_op1_op2_stub(dasm_State **Dst)
{
|->exception_handler_free_op1_op2:
| UNDEF_OPLINE_RESULT_IF_USED
| test byte OP:RX->op1_type, (IS_TMP_VAR|IS_VAR)
| je >9
| mov eax, dword OP:RX->op1.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR)
| je >9
| mov eax, dword OP:RX->op2.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| jmp ->exception_handler
return 1;
}
static int zend_jit_exception_handler_free_op2_stub(dasm_State **Dst)
{
|->exception_handler_free_op2:
| MEM_LOAD_ZTS RX, aword, executor_globals, opline_before_exception, r0
| UNDEF_OPLINE_RESULT_IF_USED
| test byte OP:RX->op2_type, (IS_TMP_VAR|IS_VAR)
| je >9
| mov eax, dword OP:RX->op2.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| jmp ->exception_handler
return 1;
}
static int zend_jit_leave_function_stub(dasm_State **Dst)
{
|->leave_function_handler:
| mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)]
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| test FCARG1d, ZEND_CALL_TOP
| jnz >1
| EXT_CALL zend_jit_leave_nested_func_helper, r0
| ADD_HYBRID_SPAD
| JMP_IP
|1:
| EXT_CALL zend_jit_leave_top_func_helper, r0
| ADD_HYBRID_SPAD
| JMP_IP
} else {
if (GCC_GLOBAL_REGS) {
| add r4, SPAD
} else {
| mov FCARG2a, FP
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD
}
| test FCARG1d, ZEND_CALL_TOP
| jnz >1
| EXT_JMP zend_jit_leave_nested_func_helper, r0
|1:
| EXT_JMP zend_jit_leave_top_func_helper, r0
}
return 1;
}
static int zend_jit_leave_throw_stub(dasm_State **Dst)
{
|->leave_throw_handler:
| // if (opline->opcode != ZEND_HANDLE_EXCEPTION) {
if (GCC_GLOBAL_REGS) {
| cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION
| je >5
| // EG(opline_before_exception) = opline;
| MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0
|5:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op
| mov aword EX->opline, IP
| // HANDLE_EXCEPTION()
| jmp ->exception_handler
} else {
| GET_IP FCARG1a
| cmp byte OP:FCARG1a->opcode, ZEND_HANDLE_EXCEPTION
| je >5
| // EG(opline_before_exception) = opline;
| MEM_STORE_ZTS aword, executor_globals, opline_before_exception, FCARG1a, r0
|5:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 2 // ZEND_VM_LEAVE
| ret
}
return 1;
}
static int zend_jit_icall_throw_stub(dasm_State **Dst)
{
|->icall_throw_handler:
| // zend_rethrow_exception(zend_execute_data *execute_data)
| mov IP, aword EX->opline
| // if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) {
| cmp byte OP:IP->opcode, ZEND_HANDLE_EXCEPTION
| je >1
| // EG(opline_before_exception) = opline;
| MEM_STORE_ZTS aword, executor_globals, opline_before_exception, IP, r0
|1:
| // opline = EG(exception_op);
| LOAD_IP_ADDR_ZTS executor_globals, exception_op
|| if (GCC_GLOBAL_REGS) {
| mov aword EX->opline, IP
|| }
| // HANDLE_EXCEPTION()
| jmp ->exception_handler
return 1;
}
static int zend_jit_throw_cannot_pass_by_ref_stub(dasm_State **Dst)
{
|->throw_cannot_pass_by_ref:
| mov r0, EX->opline
| mov ecx, dword OP:r0->result.var
| SET_Z_TYPE_INFO RX+r1, IS_UNDEF
| // last EX(call) frame may be delayed
| cmp RX, EX->call
| je >1
| mov r1, EX->call
| mov EX:RX->prev_execute_data, r1
| mov EX->call, RX
|1:
| mov RX, r0
| mov FCARG1d, dword OP:r0->op2.num
| EXT_CALL zend_cannot_pass_by_reference, r0
| cmp byte OP:RX->op1_type, IS_TMP_VAR
| jne >9
| mov eax, dword OP:RX->op1.var
| add r0, FP
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, 0, NULL
|9:
| jmp ->exception_handler
return 1;
}
static int zend_jit_undefined_offset_ex_stub(dasm_State **Dst)
{
|->undefined_offset_ex:
| SAVE_IP
| jmp ->undefined_offset
return 1;
}
static int zend_jit_undefined_offset_stub(dasm_State **Dst)
{
|->undefined_offset:
|| if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
|| }
| EXT_JMP zend_jit_undefined_long_key, r0
return 1;
}
static int zend_jit_undefined_index_ex_stub(dasm_State **Dst)
{
|->undefined_index_ex:
| SAVE_IP
| jmp ->undefined_index
return 1;
}
static int zend_jit_undefined_index_stub(dasm_State **Dst)
{
|->undefined_index:
|| if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
|| }
| EXT_JMP zend_jit_undefined_string_key, r0
return 1;
}
static int zend_jit_cannot_add_element_ex_stub(dasm_State **Dst)
{
|->cannot_add_element_ex:
| SAVE_IP
| jmp ->cannot_add_element
return 1;
}
static int zend_jit_cannot_add_element_stub(dasm_State **Dst)
{
|->cannot_add_element:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
| mov r0, EX->opline
| cmp byte OP:r0->result_type, IS_UNUSED
| jz >1
| mov eax, dword OP:r0->result.var
| SET_Z_TYPE_INFO FP + r0, IS_NULL
|1:
|.if X64WIN
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
| EXT_CALL zend_throw_error, r0
| add r4, 0x28
|.elif X64
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Cannot add element to the array as the next element is already occupied"
| EXT_CALL zend_throw_error, r0
| add r4, 8
|.else
| sub r4, 8
| push "Cannot add element to the array as the next element is already occupied"
| push 0
| EXT_CALL zend_throw_error, r0
| add r4, 28
|.endif
| ret
return 1;
}
static int zend_jit_undefined_function_stub(dasm_State **Dst)
{
|->undefined_function:
| mov r0, aword EX->opline
|.if X64
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Call to undefined function %s()"
| movsxd CARG3, dword [r0 + offsetof(zend_op, op2.constant)]
| mov CARG3, aword [r0 + CARG3]
| add CARG3, offsetof(zend_string, val)
| EXT_CALL zend_throw_error, r0
|.else
| mov r0, aword [r0 + offsetof(zend_op, op2.zv)]
| mov r0, aword [r0]
| add r0, offsetof(zend_string, val)
| mov aword A3, r0
| mov aword A2, "Call to undefined function %s()"
| mov aword A1, 0
| EXT_CALL zend_throw_error, r0
|.endif
| jmp ->exception_handler
return 1;
}
static int zend_jit_negative_shift_stub(dasm_State **Dst)
{
|->negative_shift:
| mov RX, EX->opline
|.if X64
|.if WIN
| LOAD_ADDR CARG1, &zend_ce_arithmetic_error
| mov CARG1, aword [CARG1]
|.else
| LOAD_ADDR CARG1, zend_ce_arithmetic_error
|.endif
| LOAD_ADDR CARG2, "Bit shift by negative number"
| EXT_CALL zend_throw_error, r0
|.else
| sub r4, 8
| push "Bit shift by negative number"
|.if WIN
| LOAD_ADDR r0, &zend_ce_arithmetic_error
| push aword [r0]
|.else
| PUSH_ADDR zend_ce_arithmetic_error, r0
|.endif
| EXT_CALL zend_throw_error, r0
| add r4, 16
|.endif
| jmp ->exception_handler_free_op1_op2
return 1;
}
static int zend_jit_mod_by_zero_stub(dasm_State **Dst)
{
|->mod_by_zero:
| mov RX, EX->opline
|.if X64
|.if WIN
| LOAD_ADDR CARG1, &zend_ce_division_by_zero_error
| mov CARG1, aword [CARG1]
|.else
| LOAD_ADDR CARG1, zend_ce_division_by_zero_error
|.endif
| LOAD_ADDR CARG2, "Modulo by zero"
| EXT_CALL zend_throw_error, r0
|.else
| sub r4, 8
| push "Modulo by zero"
|.if WIN
| LOAD_ADDR r0, &zend_ce_division_by_zero_error
| push aword [r0]
|.else
| PUSH_ADDR zend_ce_division_by_zero_error, r0
|.endif
| EXT_CALL zend_throw_error, r0
| add r4, 16
|.endif
| jmp ->exception_handler_free_op1_op2
return 1;
}
static int zend_jit_invalid_this_stub(dasm_State **Dst)
{
|->invalid_this:
| UNDEF_OPLINE_RESULT
|.if X64
| xor CARG1, CARG1
| LOAD_ADDR CARG2, "Using $this when not in object context"
| EXT_CALL zend_throw_error, r0
|.else
| sub r4, 8
| push "Using $this when not in object context"
| push 0
| EXT_CALL zend_throw_error, r0
| add r4, 16
|.endif
| jmp ->exception_handler
return 1;
}
static int zend_jit_double_one_stub(dasm_State **Dst)
{
|->one:
|.dword 0, 0x3ff00000
return 1;
}
static int zend_jit_hybrid_runtime_jit_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_runtime_jit:
| EXT_CALL zend_runtime_jit, r0
| JMP_IP
return 1;
}
static int zend_jit_hybrid_profile_jit_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_profile_jit:
| // ++zend_jit_profile_counter;
| .if X64
| LOAD_ADDR r0, &zend_jit_profile_counter
| inc aword [r0]
| .else
| inc aword [&zend_jit_profile_counter]
| .endif
| // op_array = (zend_op_array*)EX(func);
| mov r0, EX->func
| // run_time_cache = EX(run_time_cache);
| mov r2, EX->run_time_cache
| // jit_extension = (const void*)ZEND_FUNC_INFO(op_array);
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| // ++ZEND_COUNTER_INFO(op_array)
| inc aword [r2 + zend_jit_profile_counter_rid * sizeof(void*)]
| // return ((zend_vm_opcode_handler_t)jit_extension->orig_handler)()
| jmp aword [r0 + offsetof(zend_jit_op_array_extension, orig_handler)]
return 1;
}
static int zend_jit_hybrid_hot_code_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_hot_code:
| mov word [r2], ZEND_JIT_COUNTER_INIT
| mov FCARG1a, FP
| GET_IP FCARG2a
| EXT_CALL zend_jit_hot_func, r0
| JMP_IP
return 1;
}
/*
* This code is based Mike Pall's "Hashed profile counters" idea, implemented
* in LuaJIT. The full description may be found in "LuaJIT 2.0 intellectual
* property disclosure and research opportunities" email
* at http://lua-users.org/lists/lua-l/2009-11/msg00089.html
*
* In addition we use a variation of Knuth's multiplicative hash function
* described at https://code.i-harness.com/en/q/a21ce
*
* uint64_t hash(uint64_t x) {
* x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9;
* x = (x ^ (x >> 27)) * 0x94d049bb133111eb;
* x = x ^ (x >> 31);
* return x;
* }
*
* uint_32_t hash(uint32_t x) {
* x = ((x >> 16) ^ x) * 0x45d9f3b;
* x = ((x >> 16) ^ x) * 0x45d9f3b;
* x = (x >> 16) ^ x;
* return x;
* }
*
*/
static int zend_jit_hybrid_hot_counter_stub(dasm_State **Dst, uint32_t cost)
{
| ENDBR
| mov r0, EX->func
| mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r2, aword [r1 + offsetof(zend_jit_op_array_hot_extension, counter)]
| sub word [r2], cost
| jle ->hybrid_hot_code
| GET_IP r2
| sub r2, aword [r0 + offsetof(zend_op_array, opcodes)]
| // divide by sizeof(zend_op)
| .if X64
|| ZEND_ASSERT(sizeof(zend_op) == 32);
| sar r2, 2
| .else
|| ZEND_ASSERT(sizeof(zend_op) == 28);
| imul r2, 0xb6db6db7
| .endif
| .if X64
| jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
| .else
| jmp aword [r1+r2+offsetof(zend_jit_op_array_hot_extension, orig_handlers)]
| .endif
return 1;
}
static int zend_jit_hybrid_func_hot_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 1;
}
|->hybrid_func_hot_counter:
return zend_jit_hybrid_hot_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_loop_hot_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 1;
}
|->hybrid_loop_hot_counter:
return zend_jit_hybrid_hot_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_hybrid_hot_trace_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID) {
return 1;
}
|->hybrid_hot_trace:
| mov word [r2], ZEND_JIT_COUNTER_INIT
| mov FCARG1a, FP
| GET_IP FCARG2a
| EXT_CALL zend_jit_trace_hot_root, r0
| test eax, eax // TODO : remove this check at least for HYBRID VM ???
| jl >1
| MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0
| LOAD_IP
| JMP_IP
|1:
| EXT_JMP zend_jit_halt_op->handler, r0
return 1;
}
static int zend_jit_hybrid_trace_counter_stub(dasm_State **Dst, uint32_t cost)
{
| ENDBR
| mov r0, EX->func
| mov r1, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r1, aword [r1 + offsetof(zend_jit_op_array_trace_extension, offset)]
| mov r2, aword [IP + r1 + offsetof(zend_op_trace_info, counter)]
| sub word [r2], cost
| jle ->hybrid_hot_trace
| jmp aword [IP + r1]
return 1;
}
static int zend_jit_hybrid_func_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) {
return 1;
}
|->hybrid_func_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func)));
}
static int zend_jit_hybrid_ret_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) {
return 1;
}
|->hybrid_ret_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return)));
}
static int zend_jit_hybrid_loop_trace_counter_stub(dasm_State **Dst)
{
if (zend_jit_vm_kind != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) {
return 1;
}
|->hybrid_loop_trace_counter:
return zend_jit_hybrid_trace_counter_stub(Dst,
((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop)));
}
static int zend_jit_trace_halt_stub(dasm_State **Dst)
{
|->trace_halt:
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| EXT_JMP zend_jit_halt_op->handler, r0
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| xor IP, IP // PC must be zero
| ret
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, -1 // ZEND_VM_RETURN
| ret
}
return 1;
}
static int zend_jit_trace_exit_stub(dasm_State **Dst)
{
|->trace_exit:
|
| // Save CPU registers
|.if X64
| sub r4, 16*8+16*8-8 /* CPU regs + SSE regs */
| mov aword [r4+15*8], r15
| mov aword [r4+11*8], r11
| mov aword [r4+10*8], r10
| mov aword [r4+9*8], r9
| mov aword [r4+8*8], r8
| mov aword [r4+7*8], rdi
| mov aword [r4+6*8], rsi
| mov aword [r4+2*8], rdx
| mov aword [r4+1*8], rcx
| mov aword [r4+0*8], rax
| mov FCARG1a, aword [r4+16*8+16*8-8] // exit_num = POP
| mov FCARG2a, r4
| movsd qword [r4+16*8+15*8], xmm15
| movsd qword [r4+16*8+14*8], xmm14
| movsd qword [r4+16*8+13*8], xmm13
| movsd qword [r4+16*8+12*8], xmm12
| movsd qword [r4+16*8+11*8], xmm11
| movsd qword [r4+16*8+10*8], xmm10
| movsd qword [r4+16*8+9*8], xmm9
| movsd qword [r4+16*8+8*8], xmm8
| movsd qword [r4+16*8+7*8], xmm7
| movsd qword [r4+16*8+6*8], xmm6
| movsd qword [r4+16*8+5*8], xmm5
| movsd qword [r4+16*8+4*8], xmm4
| movsd qword [r4+16*8+3*8], xmm3
| movsd qword [r4+16*8+2*8], xmm2
| movsd qword [r4+16*8+1*8], xmm1
| movsd qword [r4+16*8+0*8], xmm0
|.if X64WIN
| sub r4, 32 /* shadow space */
|.endif
|.else
| sub r4, 8*4+8*8-4 /* CPU regs + SSE regs */
| mov aword [r4+7*4], edi
| mov aword [r4+2*4], edx
| mov aword [r4+1*4], ecx
| mov aword [r4+0*4], eax
| mov FCARG1a, aword [r4+8*4+8*8-4] // exit_num = POP
| mov FCARG2a, r4
| movsd qword [r4+8*4+7*8], xmm7
| movsd qword [r4+8*4+6*8], xmm6
| movsd qword [r4+8*4+5*8], xmm5
| movsd qword [r4+8*4+4*8], xmm4
| movsd qword [r4+8*4+3*8], xmm3
| movsd qword [r4+8*4+2*8], xmm2
| movsd qword [r4+8*4+1*8], xmm1
| movsd qword [r4+8*4+0*8], xmm0
|.endif
|
| // EX(opline) = opline
| SAVE_IP
| // zend_jit_trace_exit(trace_num, exit_num)
| EXT_CALL zend_jit_trace_exit, r0
|.if X64WIN
| add r4, 16*8+16*8+32 /* CPU regs + SSE regs + shadow space */
|.elif X64
| add r4, 16*8+16*8 /* CPU regs + SSE regs */
|.else
| add r4, 8*4+8*8 /* CPU regs + SSE regs */
|.endif
| test eax, eax
| jne >1
| // execute_data = EG(current_execute_data)
| MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0
| // opline = EX(opline)
| LOAD_IP
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
|1:
| jl ->trace_halt
| // execute_data = EG(current_execute_data)
| MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0
| // opline = EX(opline)
| LOAD_IP
| // check for interrupt (try to avoid this ???)
| MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0
| jne ->interrupt_handler
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
} else {
| mov IP, aword EX->opline
| mov FCARG1a, FP
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| call aword [IP + r0]
| test eax, eax
| jl ->trace_halt
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
return 1;
}
static int zend_jit_trace_escape_stub(dasm_State **Dst)
{
|->trace_escape:
|
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
return 1;
}
/* Keep 32 exit points in a single code block */
#define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes
#define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points
static int zend_jit_trace_exit_group_stub(dasm_State **Dst, uint32_t n)
{
uint32_t i;
for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP - 1; i++) {
| push byte i
| .byte 0xeb, (4*(ZEND_JIT_EXIT_POINTS_PER_GROUP-i)-6) // jmp >1
}
| push byte i
|// 1:
| add aword [r4], n
| jmp ->trace_exit
return 1;
}
#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call_stub(dasm_State **Dst)
{
|->context_threaded_call:
| pop r0
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| jmp aword [IP]
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| jmp aword [IP]
} else {
ZEND_UNREACHABLE();
// TODO: context threading can't work without GLOBAL REGS because we have to change
// the value of execute_data in execute_ex()
| mov FCARG1a, FP
| mov r0, aword [FP]
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| jmp aword [r0]
}
return 1;
}
#endif
static int zend_jit_assign_const_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
|->assign_const:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CONST, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_tmp_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN;
|->assign_tmp:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_TMP_VAR, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_var_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF;
|->assign_var:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_VAR, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_cv_noref_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/;
|->assign_cv_noref:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static int zend_jit_assign_cv_stub(dasm_State **Dst)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/;
|->assign_cv:
|.if X64WIN
| sub r4, 0x28
|.elif X64
| sub r4, 8
|.else
| sub r4, 12
|.endif
if (!zend_jit_assign_to_variable(
Dst, NULL,
var_addr, var_addr, -1, -1,
IS_CV, val_addr, val_info,
0, 0)) {
return 0;
}
|.if X64WIN
| add r4, 0x28
|.elif X64
| add r4, 8
|.else
| add r4, 12
|.endif
| ret
return 1;
}
static const zend_jit_stub zend_jit_stubs[] = {
JIT_STUB(interrupt_handler, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler_undef, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler_free_op1_op2, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(exception_handler_free_op2, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(leave_function, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(leave_throw, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(icall_throw, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(throw_cannot_pass_by_ref, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_offset, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_index, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(cannot_add_element, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_offset_ex, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_index_ex, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(cannot_add_element_ex, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(undefined_function, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(negative_shift, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(mod_by_zero, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(invalid_this, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(trace_halt, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(trace_exit, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(trace_escape, SP_ADJ_JIT, SP_ADJ_VM),
JIT_STUB(hybrid_runtime_jit, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_profile_jit, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_hot_code, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_func_hot_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_loop_hot_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_hot_trace, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_func_trace_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_ret_trace_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(hybrid_loop_trace_counter, SP_ADJ_VM, SP_ADJ_NONE),
JIT_STUB(assign_const, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_tmp, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_var, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_cv_noref, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(assign_cv, SP_ADJ_RET, SP_ADJ_ASSIGN),
JIT_STUB(double_one, SP_ADJ_NONE, SP_ADJ_NONE),
#ifdef CONTEXT_THREADED_JIT
JIT_STUB(context_threaded_call, SP_ADJ_RET, SP_ADJ_NONE),
#endif
};
#if ZTS && defined(ZEND_WIN32)
extern uint32_t _tls_index;
extern char *_tls_start;
extern char *_tls_end;
#endif
#ifdef HAVE_GDB
typedef struct _Unwind_Context _Unwind_Context;
typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *);
extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *);
extern uintptr_t _Unwind_GetCFA(_Unwind_Context *);
typedef struct _zend_jit_unwind_arg {
int cnt;
uintptr_t cfa[3];
} zend_jit_unwind_arg;
static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a)
{
zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a;
arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx);
arg->cnt++;
if (arg->cnt == 3) {
return 5; // _URC_END_OF_STACK
}
return 0; // _URC_NO_REASON;
}
static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data)
{
zend_jit_unwind_arg arg;
memset(&arg, 0, sizeof(arg));
_Unwind_Backtrace(zend_jit_unwind_cb, &arg);
if (arg.cnt == 3) {
sp_adj[SP_ADJ_VM] = arg.cfa[2] - arg.cfa[1];
}
}
extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data);
static zend_never_inline void zend_jit_set_sp_adj_vm(void)
{
void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *);
orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data;
zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data;
execute_ex(NULL); // set sp_adj[SP_ADJ_VM]
zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data;
}
#endif
#if defined(__APPLE__) && defined(__x86_64__)
/* Thunk format used since dydl 1284 (approx. MacOS 15)
* https://github.com/apple-oss-distributions/dyld/blob/9307719dd8dc9b385daa412b03cfceb897b2b398/libdyld/ThreadLocalVariables.h#L146 */
struct TLV_Thunkv2
{
void* func;
uint32_t key;
uint32_t offset;
};
/* Thunk format used in earlier versions */
struct TLV_Thunkv1
{
void* func;
size_t key;
size_t offset;
};
#endif
static int zend_jit_setup(void)
{
if (!zend_cpu_supports_sse2()) {
zend_error(E_CORE_ERROR, "CPU doesn't support SSE2");
return FAILURE;
}
allowed_opt_flags = 0;
if (zend_cpu_supports_avx()) {
allowed_opt_flags |= ZEND_JIT_CPU_AVX;
}
#if ZTS
# ifdef _WIN64
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void****)__readgsqword(0x58))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
// TODO: error message ???
return FAILURE;
}
} while(0);
# elif ZEND_WIN32
tsrm_tls_index = _tls_index * sizeof(void*);
/* To find offset of "_tsrm_ls_cache" in TLS segment we perform a linear scan of local TLS memory */
/* Probably, it might be better solution */
do {
void ***tls_mem = ((void****)__readfsdword(0x2c))[_tls_index];
void *val = _tsrm_ls_cache;
size_t offset = 0;
size_t size = (char*)&_tls_end - (char*)&_tls_start;
while (offset < size) {
if (*tls_mem == val) {
tsrm_tls_offset = offset;
break;
}
tls_mem++;
offset += sizeof(void*);
}
if (offset >= size) {
// TODO: error message ???
return FAILURE;
}
} while(0);
# elif defined(__APPLE__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
struct TLV_Thunkv2 *thunk;
__asm__(
"leaq __tsrm_ls_cache(%%rip),%0"
: "=r" (thunk));
/* Detect dyld 1284: With dyld 1284, thunk->func will be _tlv_get_addr.
* Unfortunately this symbol is private, but we can find it
* as _tlv_bootstrap+8: https://github.com/apple-oss-distributions/dyld/blob/9307719dd8dc9b385daa412b03cfceb897b2b398/libdyld/threadLocalHelpers.s#L54
* In earlier versions, thunk->func will be tlv_get_addr, which is not
* _tlv_bootstrap+8.
*/
if (thunk->func == (void*)((char*)_tlv_bootstrap + 8)) {
tsrm_tls_offset = thunk->offset;
tsrm_tls_index = (size_t)thunk->key * 8;
} else {
struct TLV_Thunkv1 *thunkv1 = (struct TLV_Thunkv1*) thunk;
tsrm_tls_offset = thunkv1->offset;
tsrm_tls_index = thunkv1->key * 8;
}
}
# elif defined(__GNUC__) && defined(__x86_64__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if defined(__has_attribute) && __has_attribute(tls_model) && !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("movq _tsrm_ls_cache@gottpoff(%%rip),%0"
: "=r" (ret));
tsrm_ls_cache_tcb_offset = ret;
#elif defined(__MUSL__)
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=D" (ti));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 8;
#elif defined(__FreeBSD__)
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=D" (ti));
tsrm_tls_offset = ti[1];
/* Index is offset by 1 on FreeBSD (https://github.com/freebsd/freebsd-src/blob/bf56e8b9c8639ac4447d223b83cdc128107cc3cd/libexec/rtld-elf/rtld.c#L5260) */
tsrm_tls_index = (ti[0] + 1) * 8;
#else
size_t *ti;
__asm__(
"leaq _tsrm_ls_cache@tlsgd(%%rip), %0\n"
: "=D" (ti));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 16;
#endif
}
# elif defined(__GNUC__) && defined(__i386__)
tsrm_ls_cache_tcb_offset = tsrm_get_ls_cache_tcb_offset();
if (tsrm_ls_cache_tcb_offset == 0) {
#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__MUSL__)
size_t ret;
asm ("leal _tsrm_ls_cache@ntpoff,%0\n"
: "=a" (ret));
tsrm_ls_cache_tcb_offset = ret;
#else
size_t *ti, _ebx, _ecx, _edx;
__asm__(
"call 1f\n"
".subsection 1\n"
"1:\tmovl (%%esp), %%ebx\n\t"
"ret\n"
".previous\n\t"
"addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n\t"
"call ___tls_get_addr@plt\n\t"
"leal _tsrm_ls_cache@tlsldm(%%ebx), %0\n"
: "=a" (ti), "=&b" (_ebx), "=&c" (_ecx), "=&d" (_edx));
tsrm_tls_offset = ti[1];
tsrm_tls_index = ti[0] * 8;
#endif
}
# endif
#endif
memset(sp_adj, 0, sizeof(sp_adj));
#ifdef HAVE_GDB
sp_adj[SP_ADJ_RET] = sizeof(void*);
|.if X64WIN
|| sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 0x28; // sub r4, 0x28
|.elif X64
|| sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 8; // sub r4, 8
|.else
|| sp_adj[SP_ADJ_ASSIGN] = sp_adj[SP_ADJ_RET] + 12; // sub r4, 12
|.endif
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
zend_jit_set_sp_adj_vm(); // set sp_adj[SP_ADJ_VM]
#ifndef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM] + HYBRID_SPAD; // sub r4, HYBRID_SPAD
#else
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_VM];
#endif
} else if (GCC_GLOBAL_REGS) {
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + SPAD; // sub r4, SPAD
} else {
|| sp_adj[SP_ADJ_JIT] = sp_adj[SP_ADJ_RET] + NR_SPAD; // sub r4, NR_SPAD
}
#endif
return SUCCESS;
}
static ZEND_ATTRIBUTE_UNUSED int zend_jit_trap(dasm_State **Dst)
{
| int3
return 1;
}
static int zend_jit_align_func(dasm_State **Dst)
{
reuse_ip = 0;
delayed_call_chain = 0;
last_valid_opline = NULL;
use_last_vald_opline = 0;
track_last_valid_opline = 0;
jit_return_label = -1;
|.align 16
return 1;
}
static int zend_jit_align_stub(dasm_State **Dst)
{
|.align 16
return 1;
}
static int zend_jit_prologue(dasm_State **Dst)
{
| ENDBR
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| SUB_HYBRID_SPAD
} else if (GCC_GLOBAL_REGS) {
| sub r4, SPAD // stack alignment
} else {
| sub r4, NR_SPAD // stack alignment
| mov aword T2, FP // save FP
| mov aword T3, RX // save IP
| mov FP, FCARG1a
}
return 1;
}
static int zend_jit_label(dasm_State **Dst, unsigned int label)
{
|=>label:
return 1;
}
static int zend_jit_save_call_chain(dasm_State **Dst, uint32_t call_level)
{
| // call->prev_execute_data = EX(call);
if (call_level == 1) {
| mov aword EX:RX->prev_execute_data, 0
} else {
| mov r0, EX->call
| mov EX:RX->prev_execute_data, r0
}
| // EX(call) = call;
| mov EX->call, RX
delayed_call_chain = 0;
return 1;
}
static int zend_jit_set_ip(dasm_State **Dst, const zend_op *opline)
{
if (last_valid_opline == opline) {
zend_jit_use_last_valid_opline();
} else if (GCC_GLOBAL_REGS && last_valid_opline) {
zend_jit_use_last_valid_opline();
| ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
} else {
| LOAD_IP_ADDR opline
}
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_set_ip_ex(dasm_State **Dst, const zend_op *opline, bool set_ip_reg)
{
if (last_valid_opline == opline) {
zend_jit_use_last_valid_opline();
} else if (GCC_GLOBAL_REGS && last_valid_opline) {
zend_jit_use_last_valid_opline();
| ADD_IP (opline - last_valid_opline) * sizeof(zend_op);
} else if (!GCC_GLOBAL_REGS && set_ip_reg) {
| LOAD_ADDR RX, opline
| mov aword EX->opline, RX
} else {
| LOAD_IP_ADDR opline
}
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_set_valid_ip(dasm_State **Dst, const zend_op *opline)
{
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (!zend_jit_set_ip(Dst, opline)) {
return 0;
}
reuse_ip = 0;
return 1;
}
static int zend_jit_check_timeout(dasm_State **Dst, const zend_op *opline, const void *exit_addr)
{
#if 0
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
| MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0
| jne ->interrupt_handler
#else
| MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0
if (exit_addr) {
| jne &exit_addr
} else if (last_valid_opline == opline) {
|| zend_jit_use_last_valid_opline();
| jne ->interrupt_handler
} else {
| jne >1
|.cold_code
|1:
| LOAD_IP_ADDR opline
| jmp ->interrupt_handler
|.code
}
#endif
return 1;
}
static int zend_jit_trace_end_loop(dasm_State **Dst, int loop_label, const void *timeout_exit_addr)
{
if (timeout_exit_addr) {
| MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0
| je =>loop_label
| jmp &timeout_exit_addr
} else {
| jmp =>loop_label
}
return 1;
}
static int zend_jit_check_exception(dasm_State **Dst)
{
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| jne ->exception_handler
return 1;
}
static int zend_jit_check_exception_undef_result(dasm_State **Dst, const zend_op *opline)
{
if (opline->result_type & (IS_TMP_VAR|IS_VAR)) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| jne ->exception_handler_undef
return 1;
}
return zend_jit_check_exception(Dst);
}
static int zend_jit_trace_begin(dasm_State **Dst, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num)
{
zend_regset regset = ZEND_REGSET_SCRATCH;
#if ZTS
if (1) {
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(jit_trace_num)))) {
#endif
/* assignment to EG(jit_trace_num) shouldn't clober CPU register used by deoptimizer */
if (parent) {
int i;
int parent_vars_count = parent->exit_info[exit_num].stack_size;
zend_jit_trace_stack *parent_stack =
parent->stack_map +
parent->exit_info[exit_num].stack_offset;
for (i = 0; i < parent_vars_count; i++) {
if (STACK_REG(parent_stack, i) != ZREG_NONE) {
if (STACK_REG(parent_stack, i) < ZREG_NUM) {
ZEND_REGSET_EXCL(regset, STACK_REG(parent_stack, i));
} else if (STACK_REG(parent_stack, i) == ZREG_ZVAL_COPY_GPR0) {
ZEND_REGSET_EXCL(regset, ZREG_R0);
}
}
}
}
}
if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) {
ZEND_REGSET_EXCL(regset, ZREG_R0);
}
current_trace_num = trace_num;
| // EG(jit_trace_num) = trace_num;
if (regset == ZEND_REGSET_EMPTY) {
| push r0
| MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, r0
| pop r0
} else {
zend_reg tmp = ZEND_REGSET_FIRST(regset);
| MEM_STORE_ZTS dword, executor_globals, jit_trace_num, trace_num, Ra(tmp)
(void)tmp;
}
return 1;
}
static int zend_jit_trace_end(dasm_State **Dst, zend_jit_trace_info *t)
{
|.cold_code
|=>1: // end of the code
|.code
return 1;
}
/* This taken from LuaJIT. Thanks to Mike Pall. */
static uint32_t _asm_x86_inslen(const uint8_t* p)
{
static const uint8_t map_op1[256] = {
0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x20,
0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x51,0x51,
0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,0x92,0x92,0x92,0x92,0x52,0x45,0x10,0x51,
#if defined(__x86_64__) || defined(_M_X64)
0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x10,0x14,0x14,0x14,0x14,0x14,0x14,0x14,0x14,
#else
0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
#endif
0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,
0x51,0x51,0x92,0x92,0x10,0x10,0x12,0x11,0x45,0x86,0x52,0x93,0x51,0x51,0x51,0x51,
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
0x93,0x86,0x93,0x93,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x51,0x47,0x51,0x51,0x51,0x51,0x51,
#if defined(__x86_64__) || defined(_M_X64)
0x59,0x59,0x59,0x59,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
#else
0x55,0x55,0x55,0x55,0x51,0x51,0x51,0x51,0x52,0x45,0x51,0x51,0x51,0x51,0x51,0x51,
#endif
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x05,0x05,0x05,0x05,0x05,0x05,0x05,0x05,
0x93,0x93,0x53,0x51,0x70,0x71,0x93,0x86,0x54,0x51,0x53,0x51,0x51,0x52,0x51,0x51,
0x92,0x92,0x92,0x92,0x52,0x52,0x51,0x51,0x92,0x92,0x92,0x92,0x92,0x92,0x92,0x92,
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x45,0x45,0x47,0x52,0x51,0x51,0x51,0x51,
0x10,0x51,0x10,0x10,0x51,0x51,0x63,0x66,0x51,0x51,0x51,0x51,0x51,0x51,0x92,0x92
};
static const uint8_t map_op2[256] = {
0x93,0x93,0x93,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x51,0x52,0x51,0x93,0x52,0x94,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x34,0x51,0x35,0x51,0x51,0x51,0x51,0x51,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x53,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x94,0x54,0x54,0x54,0x93,0x93,0x93,0x52,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,0x46,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x52,0x52,0x52,0x93,0x94,0x93,0x51,0x51,0x52,0x52,0x52,0x93,0x94,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x94,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x94,0x93,0x94,0x94,0x94,0x93,0x52,0x52,0x52,0x52,0x52,0x52,0x52,0x52,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,
0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x93,0x52
};
uint32_t result = 0;
uint32_t prefixes = 0;
uint32_t x = map_op1[*p];
for (;;) {
switch (x >> 4) {
case 0:
return result + x + (prefixes & 4);
case 1:
prefixes |= x;
x = map_op1[*++p];
result++;
break;
case 2:
x = map_op2[*++p];
break;
case 3:
p++;
goto mrm;
case 4:
result -= (prefixes & 2);
/* fallthrough */
case 5:
return result + (x & 15);
case 6: /* Group 3. */
if (p[1] & 0x38) {
x = 2;
} else if ((prefixes & 2) && (x == 0x66)) {
x = 4;
}
goto mrm;
case 7: /* VEX c4/c5. */
#if !defined(__x86_64__) && !defined(_M_X64)
if (p[1] < 0xc0) {
x = 2;
goto mrm;
}
#endif
if (x == 0x70) {
x = *++p & 0x1f;
result++;
if (x >= 2) {
p += 2;
result += 2;
goto mrm;
}
}
p++;
result++;
x = map_op2[*++p];
break;
case 8:
result -= (prefixes & 2);
/* fallthrough */
case 9:
mrm:
/* ModR/M and possibly SIB. */
result += (x & 15);
x = *++p;
switch (x >> 6) {
case 0:
if ((x & 7) == 5) {
return result + 4;
}
break;
case 1:
result++;
break;
case 2:
result += 4;
break;
case 3:
return result;
}
if ((x & 7) == 4) {
result++;
if (x < 0x40 && (p[1] & 7) == 5) {
result += 4;
}
}
return result;
}
}
}
typedef ZEND_SET_ALIGNED(1, uint16_t unaligned_uint16_t);
typedef ZEND_SET_ALIGNED(1, int32_t unaligned_int32_t);
static int zend_jit_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr)
{
int ret = 0;
uint8_t *p, *end;
if (jmp_table_size) {
const void **jmp_slot = (const void **)((char*)code + ZEND_MM_ALIGNED_SIZE_EX(size, sizeof(void*)));
do {
if (*jmp_slot == from_addr) {
*jmp_slot = to_addr;
ret++;
}
jmp_slot++;
} while (--jmp_table_size);
}
p = (uint8_t*)code;
end = p + size - 5;
while (p < end) {
if ((*(unaligned_uint16_t*)p & 0xf0ff) == 0x800f && p + *(unaligned_int32_t*)(p+2) == (uint8_t*)from_addr - 6) {
*(unaligned_int32_t*)(p+2) = ((uint8_t*)to_addr - (p + 6));
ret++;
} else if (*p == 0xe9 && p + *(unaligned_int32_t*)(p+1) == (uint8_t*)from_addr - 5) {
*(unaligned_int32_t*)(p+1) = ((uint8_t*)to_addr - (p + 5));
ret++;
}
p += _asm_x86_inslen(p);
}
#ifdef HAVE_VALGRIND
VALGRIND_DISCARD_TRANSLATIONS(code, size);
#endif
return ret;
}
static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr)
{
return zend_jit_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr);
}
static int zend_jit_trace_link_to_root(dasm_State **Dst, zend_jit_trace_info *t, const void *timeout_exit_addr)
{
const void *link_addr;
size_t prologue_size;
/* Skip prologue. */
// TODO: don't hardcode this ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
prologue_size = 0;
#elif defined(__x86_64__) || defined(_M_X64)
// sub r4, HYBRID_SPAD
prologue_size = 4;
#else
// sub r4, HYBRID_SPAD
prologue_size = 3;
#endif
} else if (GCC_GLOBAL_REGS) {
// sub r4, SPAD // stack alignment
#if defined(__x86_64__) || defined(_M_X64)
prologue_size = 4;
#else
prologue_size = 3;
#endif
} else {
// sub r4, NR_SPAD // stack alignment
// mov aword T2, FP // save FP
// mov aword T3, RX // save IP
// mov FP, FCARG1a
#if defined(__x86_64__) || defined(_M_X64)
prologue_size = 17;
#else
prologue_size = 13;
#endif
}
link_addr = (const void*)((const char*)t->code_start + prologue_size + ENDBR_PADDING);
if (timeout_exit_addr) {
/* Check timeout for links to LOOP */
| MEM_CMP_ZTS byte, executor_globals, vm_interrupt, 0, r0
| je &link_addr
| jmp &timeout_exit_addr
} else {
| jmp &link_addr
}
return 1;
}
static int zend_jit_trace_return(dasm_State **Dst, bool original_handler, const zend_op *opline)
{
#if 0
| jmp ->trace_escape
#else
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
if (!original_handler) {
| JMP_IP
} else {
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
}
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
if (!original_handler) {
| JMP_IP
} else {
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| jmp aword [IP + r0]
}
} else {
if (original_handler) {
| mov FCARG1a, FP
| mov r0, EX->func
| mov r0, aword [r0 + offsetof(zend_op_array, reserved[zend_func_info_rid])]
| mov r0, aword [r0 + offsetof(zend_jit_op_array_trace_extension, offset)]
| call aword [IP + r0]
}
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
if (!original_handler || !opline ||
(opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF
&& opline->opcode != ZEND_GENERATOR_RETURN
&& opline->opcode != ZEND_GENERATOR_CREATE
&& opline->opcode != ZEND_YIELD
&& opline->opcode != ZEND_YIELD_FROM)) {
| mov r0, 2 // ZEND_VM_LEAVE
}
| ret
}
#endif
return 1;
}
static int zend_jit_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint8_t type)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_Z_TYPE FP + var, type, &exit_addr
return 1;
}
static int zend_jit_scalar_type_guard(dasm_State **Dst, const zend_op *opline, uint32_t var)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| cmp byte [FP+var+offsetof(zval, u1.v.type)], IS_STRING
| jae &exit_addr
return 1;
}
static int zend_jit_packed_guard(dasm_State **Dst, const zend_op *opline, uint32_t var, uint32_t op_info)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| GET_ZVAL_LVAL ZREG_FCARG1, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var)
if (op_info & MAY_BE_ARRAY_PACKED) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jz &exit_addr
} else {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jnz &exit_addr
}
return 1;
}
static int zend_jit_trace_handler(dasm_State **Dst, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace)
{
zend_jit_op_array_trace_extension *jit_extension =
(zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array);
size_t offset = jit_extension->offset;
const void *handler =
(zend_vm_opcode_handler_t)ZEND_OP_TRACE_INFO(opline, offset)->call_handler;
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL handler, r0
if (may_throw
&& opline->opcode != ZEND_RETURN
&& opline->opcode != ZEND_RETURN_BY_REF) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r1
| jne ->exception_handler
}
while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) {
trace++;
}
if (!GCC_GLOBAL_REGS
&& (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_GENERATOR_CREATE) {
| MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r1
}
}
if (zend_jit_trace_may_exit(op_array, opline)) {
if (opline->opcode == ZEND_RETURN ||
opline->opcode == ZEND_RETURN_BY_REF ||
opline->opcode == ZEND_GENERATOR_CREATE) {
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
/* this check may be handled by the following OPLINE guard or jmp [IP] */
| cmp IP, zend_jit_halt_op
| je ->trace_halt
}
} else if (GCC_GLOBAL_REGS) {
| test IP, IP
| je ->trace_halt
} else {
| test eax, eax
| jl ->trace_halt
}
} else if (opline->opcode == ZEND_EXIT ||
opline->opcode == ZEND_GENERATOR_RETURN ||
opline->opcode == ZEND_YIELD ||
opline->opcode == ZEND_YIELD_FROM) {
| jmp ->trace_halt
}
if (trace->op != ZEND_JIT_TRACE_END ||
(trace->stop != ZEND_JIT_TRACE_STOP_RETURN &&
trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
const zend_op *next_opline = trace->opline;
const zend_op *exit_opline = NULL;
uint32_t exit_point;
const void *exit_addr;
uint32_t old_info = 0;
uint32_t old_res_info = 0;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
if (zend_is_smart_branch(opline)) {
bool exit_if_true = 0;
exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true);
} else {
switch (opline->opcode) {
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
case ZEND_JMP_SET:
case ZEND_COALESCE:
case ZEND_JMP_NULL:
case ZEND_FE_RESET_R:
case ZEND_FE_RESET_RW:
exit_opline = (trace->opline == opline + 1) ?
OP_JMP_ADDR(opline, opline->op2) :
opline + 1;
break;
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
exit_opline = (trace->opline == opline + 1) ?
ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) :
opline + 1;
break;
}
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1);
}
break;
}
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
}
exit_point = zend_jit_trace_get_exit_point(exit_opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
switch (opline->opcode) {
case ZEND_FE_FETCH_R:
case ZEND_FE_FETCH_RW:
if (opline->op2_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info);
}
break;
case ZEND_BIND_INIT_STATIC_OR_JMP:
if (opline->op1_type == IS_CV) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info);
}
break;
}
if (!exit_addr) {
return 0;
}
| CMP_IP next_opline
| jne &exit_addr
}
}
zend_jit_set_last_valid_opline(trace->opline);
return 1;
}
static int zend_jit_handler(dasm_State **Dst, const zend_op *opline, int may_throw)
{
const void *handler;
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
handler = zend_get_opcode_handler_func(opline);
} else {
handler = opline->handler;
}
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL handler, r0
if (may_throw) {
zend_jit_check_exception(Dst);
}
/* Skip the following OP_DATA */
switch (opline->opcode) {
case ZEND_ASSIGN_DIM:
case ZEND_ASSIGN_OBJ:
case ZEND_ASSIGN_STATIC_PROP:
case ZEND_ASSIGN_DIM_OP:
case ZEND_ASSIGN_OBJ_OP:
case ZEND_ASSIGN_STATIC_PROP_OP:
case ZEND_ASSIGN_STATIC_PROP_REF:
case ZEND_ASSIGN_OBJ_REF:
zend_jit_set_last_valid_opline(opline + 2);
break;
default:
zend_jit_set_last_valid_opline(opline + 1);
break;
}
return 1;
}
static int zend_jit_tail_handler(dasm_State **Dst, const zend_op *opline)
{
if (!zend_jit_set_valid_ip(Dst, opline)) {
return 0;
}
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
if (opline->opcode == ZEND_DO_UCALL ||
opline->opcode == ZEND_DO_FCALL_BY_NAME ||
opline->opcode == ZEND_DO_FCALL ||
opline->opcode == ZEND_RETURN) {
/* Use inlined HYBRID VM handler */
const void *handler = opline->handler;
| ADD_HYBRID_SPAD
| EXT_JMP handler, r0
} else {
const void *handler = zend_get_opcode_handler_func(opline);
| EXT_CALL handler, r0
| ADD_HYBRID_SPAD
| JMP_IP
}
} else {
const void *handler = opline->handler;
if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
} else {
| mov FCARG1a, FP
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
}
| EXT_JMP handler, r0
}
zend_jit_reset_last_valid_opline();
return 1;
}
static int zend_jit_trace_opline_guard(dasm_State **Dst, const zend_op *opline)
{
uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| CMP_IP opline
| jne &exit_addr
zend_jit_set_last_valid_opline(opline);
return 1;
}
static int zend_jit_jmp(dasm_State **Dst, unsigned int target_label)
{
| jmp =>target_label
return 1;
}
static int zend_jit_cond_jmp(dasm_State **Dst, const zend_op *next_opline, unsigned int target_label)
{
| CMP_IP next_opline
| jne =>target_label
zend_jit_set_last_valid_opline(next_opline);
return 1;
}
#ifdef CONTEXT_THREADED_JIT
static int zend_jit_context_threaded_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
if (!zend_jit_handler(Dst, opline, 1)) return 0;
if (opline->opcode == ZEND_DO_UCALL) {
| call ->context_threaded_call
} else {
const zend_op *next_opline = opline + 1;
| CMP_IP next_opline
| je =>next_block
| call ->context_threaded_call
}
return 1;
}
#endif
static int zend_jit_call(dasm_State **Dst, const zend_op *opline, unsigned int next_block)
{
#ifdef CONTEXT_THREADED_JIT
return zend_jit_context_threaded_call(Dst, opline, next_block);
#else
return zend_jit_tail_handler(Dst, opline);
#endif
}
static int zend_jit_spill_store(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type)
{
ZEND_ASSERT(Z_MODE(src) == IS_REG);
ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| SET_ZVAL_LVAL dst, Ra(Z_REG(src))
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) {
| SET_ZVAL_TYPE_INFO dst, IS_LONG
}
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| DOUBLE_SET_ZVAL_DVAL dst, Z_REG(src)
if (set_type &&
(Z_REG(dst) != ZREG_FP ||
!JIT_G(current_frame) ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) {
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_load_reg(dasm_State **Dst, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL);
ZEND_ASSERT(Z_MODE(dst) == IS_REG);
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| GET_ZVAL_LVAL Z_REG(dst), src
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| DOUBLE_GET_ZVAL_DVAL Z_REG(dst), src
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_store_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg, bool set_type)
{
zend_jit_addr src = ZEND_ADDR_REG(reg);
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
return zend_jit_spill_store(Dst, src, dst, info, set_type);
}
static int zend_jit_store_var_type(dasm_State **Dst, int var, uint32_t type)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
| SET_ZVAL_TYPE_INFO dst, type
return 1;
}
static int zend_jit_store_var_if_necessary(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
return zend_jit_spill_store(Dst, src, dst, info, 1);
}
return 1;
}
static int zend_jit_store_var_if_necessary_ex(dasm_State **Dst, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info)
{
if (Z_MODE(src) == IS_REG && Z_STORE(src)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
bool set_type = 1;
if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) ==
(old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) {
if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) {
set_type = 0;
}
}
return zend_jit_spill_store(Dst, src, dst, info, set_type);
}
return 1;
}
static int zend_jit_load_var(dasm_State **Dst, uint32_t info, int var, zend_reg reg)
{
zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
zend_jit_addr dst = ZEND_ADDR_REG(reg);
return zend_jit_load_reg(Dst, src, dst, info);
}
static int zend_jit_invalidate_var_if_necessary(dasm_State **Dst, uint8_t op_type, zend_jit_addr addr, znode_op op)
{
if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var);
| SET_ZVAL_TYPE_INFO dst, IS_UNDEF
}
return 1;
}
static int zend_jit_update_regs(dasm_State **Dst, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info)
{
if (!zend_jit_same_addr(src, dst)) {
if (Z_MODE(src) == IS_REG) {
if (Z_MODE(dst) == IS_REG) {
if ((info & MAY_BE_ANY) == MAY_BE_LONG) {
| mov Ra(Z_REG(dst)), Ra(Z_REG(src))
} else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| SSE_AVX_INS movaps, vmovaps, xmm(Z_REG(dst)-ZREG_XMM0), xmm(Z_REG(src)-ZREG_XMM0)
} else {
ZEND_UNREACHABLE();
}
if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) {
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(Dst, dst, var_addr, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else if (Z_MODE(dst) == IS_MEM_ZVAL) {
if (!Z_LOAD(src) && !Z_STORE(src)) {
if (!zend_jit_spill_store(Dst, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(src) == IS_MEM_ZVAL) {
if (Z_MODE(dst) == IS_REG) {
if (!zend_jit_load_reg(Dst, src, dst, info)) {
return 0;
}
} else {
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) {
dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
if (!zend_jit_spill_store(Dst, src, dst, info,
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN ||
(1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY)
)) {
return 0;
}
}
return 1;
}
static int zend_jit_escape_if_undef_r0(dasm_State **Dst, int var, uint32_t flags, const zend_op *opline)
{
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >1
if (flags & ZEND_JIT_EXIT_RESTORE_CALL) {
if (!zend_jit_save_call_chain(Dst, -1)) {
return 0;
}
}
ZEND_ASSERT(opline);
if ((opline-1)->opcode != ZEND_FETCH_CONSTANT
&& (opline-1)->opcode != ZEND_FETCH_LIST_R
&& ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)) {
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var);
| IF_NOT_ZVAL_REFCOUNTED val_addr, >2
| GET_ZVAL_PTR r0, val_addr
| GC_ADDREF r0
|2:
}
| LOAD_IP_ADDR (opline - 1)
| jmp ->trace_escape
|1:
return 1;
}
static int zend_jit_store_const(dasm_State **Dst, int var, zend_reg reg)
{
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var));
if (reg == ZREG_LONG_MIN_MINUS_1) {
|.if X64
| SET_ZVAL_LVAL dst, 0x00000000
| SET_ZVAL_W2 dst, 0xc3e00000
|.else
| SET_ZVAL_LVAL dst, 0x00200000
| SET_ZVAL_W2 dst, 0xc1e00000
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE
} else if (reg == ZREG_LONG_MIN) {
|.if X64
| SET_ZVAL_LVAL dst, 0x00000000
| SET_ZVAL_W2 dst, 0x80000000
|.else
| SET_ZVAL_LVAL dst, ZEND_LONG_MIN
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_LONG
} else if (reg == ZREG_LONG_MAX) {
|.if X64
| SET_ZVAL_LVAL dst, 0xffffffff
| SET_ZVAL_W2 dst, 0x7fffffff
|.else
| SET_ZVAL_LVAL dst, ZEND_LONG_MAX
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_LONG
} else if (reg == ZREG_LONG_MAX_PLUS_1) {
|.if X64
| SET_ZVAL_LVAL dst, 0
| SET_ZVAL_W2 dst, 0x43e00000
|.else
| SET_ZVAL_LVAL dst, 0
| SET_ZVAL_W2 dst, 0x41e00000
|.endif
| SET_ZVAL_TYPE_INFO dst, IS_DOUBLE
} else if (reg == ZREG_NULL) {
| SET_ZVAL_TYPE_INFO dst, IS_NULL
} else if (reg == ZREG_ZVAL_TRY_ADDREF) {
| IF_NOT_ZVAL_REFCOUNTED dst, >1
| GET_ZVAL_PTR r1, dst
| GC_ADDREF r1
|1:
} else if (reg == ZREG_ZVAL_COPY_GPR0) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| ZVAL_COPY_VALUE dst, -1, val_addr, -1, ZREG_R1, ZREG_R2
| TRY_ADDREF -1, ch, r2
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_free_trampoline(dasm_State **Dst)
{
| /// if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE))
| test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_CALL_VIA_TRAMPOLINE
| jz >1
| mov FCARG1a, r0
| EXT_CALL zend_jit_free_trampoline_helper, r0
|1:
return 1;
}
static int zend_jit_inc_dec(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, MAY_BE_LONG)) {
return 0;
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
| LONG_OP_WITH_32BIT_CONST add, op1_def_addr, Z_L(1)
} else {
| LONG_OP_WITH_32BIT_CONST sub, op1_def_addr, Z_L(1)
}
if (may_overflow &&
(((op1_def_info & MAY_BE_GUARD) && (op1_def_info & MAY_BE_LONG)) ||
((opline->result_type != IS_UNUSED && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG))))) {
int32_t exit_point;
const void *exit_addr;
zend_jit_trace_stack *stack;
uint32_t old_op1_info, old_res_info = 0;
stack = JIT_G(current_frame)->stack;
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0);
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MAX_PLUS_1);
} else {
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_LONG_MIN_MINUS_1);
}
if (opline->result_type != IS_UNUSED) {
old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
if (opline->opcode == ZEND_PRE_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX_PLUS_1);
} else if (opline->opcode == ZEND_PRE_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN_MINUS_1);
} else if (opline->opcode == ZEND_POST_INC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MAX);
} else if (opline->opcode == ZEND_POST_DEC) {
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_LONG_MIN);
}
}
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jo &exit_addr
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
if (opline->result_type != IS_UNUSED) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
}
} else if (may_overflow) {
| jo >1
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
|.cold_code
|1:
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
|.if X64
| mov64 rax, 0x43e0000000000000
| SET_ZVAL_LVAL op1_def_addr, rax
|.else
| SET_ZVAL_LVAL op1_def_addr, 0
| SET_ZVAL_W2 op1_def_addr, 0x41e00000
|.endif
} else {
|.if X64
| mov64 rax, 0xc3e0000000000000
| SET_ZVAL_LVAL op1_def_addr, rax
|.else
| SET_ZVAL_LVAL op1_def_addr, 0x00200000
| SET_ZVAL_W2 op1_def_addr, 0xc1e00000
|.endif
}
if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO op1_def_addr, IS_DOUBLE
}
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R1
}
| jmp >3
|.code
} else {
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_def_addr, MAY_BE_LONG, ZREG_R0, ZREG_R1
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
|.cold_code
|2:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| SET_EX_OPLINE opline, r0
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >2
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
op1_info |= MAY_BE_NULL;
}
|2:
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| // ZVAL_DEREF(var_ptr);
if (op1_info & MAY_BE_REF) {
| IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >2
| GET_Z_PTR FCARG1a, FCARG1a
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jz >1
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
} else {
| xor FCARG2a, FCARG2a
}
if (opline->opcode == ZEND_PRE_INC) {
| EXT_CALL zend_jit_pre_inc_typed_ref, r0
} else if (opline->opcode == ZEND_PRE_DEC) {
| EXT_CALL zend_jit_pre_dec_typed_ref, r0
} else if (opline->opcode == ZEND_POST_INC) {
| EXT_CALL zend_jit_post_inc_typed_ref, r0
} else if (opline->opcode == ZEND_POST_DEC) {
| EXT_CALL zend_jit_post_dec_typed_ref, r0
} else {
ZEND_UNREACHABLE();
}
zend_jit_check_exception(Dst);
| jmp >3
|1:
| lea FCARG1a, [FCARG1a + offsetof(zend_reference, val)]
|2:
}
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
| ZVAL_COPY_VALUE res_addr, res_use_info, val_addr, op1_info, ZREG_R0, ZREG_R2
| TRY_ADDREF op1_info, ah, r2
}
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_inc, r0
} else {
| EXT_CALL increment_function, r0
}
} else {
if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_dec, r0
} else {
| EXT_CALL decrement_function, r0
}
}
if (may_throw) {
zend_jit_check_exception(Dst);
}
} else {
zend_reg tmp_reg;
if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, MAY_BE_DOUBLE, ZREG_R0, ZREG_R2
}
if (Z_MODE(op1_def_addr) == IS_REG) {
tmp_reg = Z_REG(op1_def_addr);
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
tmp_reg = Z_REG(op1_addr);
} else {
tmp_reg = ZREG_XMM0;
}
| DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr
if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) {
if (CAN_USE_AVX()) {
| vaddsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
} else {
| addsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
}
} else {
if (CAN_USE_AVX()) {
| vsubsd xmm(tmp_reg-ZREG_XMM0), xmm(tmp_reg-ZREG_XMM0), qword [->one]
} else {
| subsd xmm(tmp_reg-ZREG_XMM0), qword [->one]
}
}
| DOUBLE_SET_ZVAL_DVAL op1_def_addr, tmp_reg
if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) &&
opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, res_use_info, op1_addr, op1_def_info, ZREG_R0, ZREG_R1
| TRY_ADDREF op1_def_info, ah, r1
}
}
| jmp >3
|.code
}
|3:
if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
static int zend_jit_opline_uses_reg(const zend_op *opline, int8_t reg)
{
if ((opline+1)->opcode == ZEND_OP_DATA
&& ((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV))
&& JIT_G(current_frame)->stack[EX_VAR_TO_NUM((opline+1)->op1.var)].reg == reg) {
return 1;
}
return
((opline->result_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->result.var)].reg == reg) ||
((opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op1.var)].reg == reg) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR|IS_CV)) &&
JIT_G(current_frame)->stack[EX_VAR_TO_NUM(opline->op2.var)].reg == reg);
}
static int zend_jit_math_long_long(dasm_State **Dst,
const zend_op *opline,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow)
{
bool must_set_cflags = 0;
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
zend_reg tmp_reg = ZREG_R0;
if (Z_MODE(res_addr) == IS_REG && (res_info & MAY_BE_LONG)) {
if (may_overflow && (res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& zend_jit_opline_uses_reg(opline, Z_REG(res_addr))) {
result_reg = ZREG_R0;
} else {
result_reg = Z_REG(res_addr);
}
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr) && !may_overflow) {
result_reg = Z_REG(op1_addr);
} else if (Z_REG(res_addr) != ZREG_R0) {
result_reg = ZREG_R0;
} else {
/* ASSIGN_DIM_OP */
result_reg = ZREG_FCARG1;
tmp_reg = ZREG_FCARG1;
}
if (may_overflow) {
must_set_cflags = 1;
} else {
const zend_op *next_opline = opline + 1;
if (next_opline->opcode == ZEND_IS_EQUAL ||
next_opline->opcode == ZEND_IS_NOT_EQUAL ||
next_opline->opcode == ZEND_IS_SMALLER ||
next_opline->opcode == ZEND_IS_SMALLER_OR_EQUAL ||
next_opline->opcode == ZEND_CASE ||
next_opline->opcode == ZEND_IS_IDENTICAL ||
next_opline->opcode == ZEND_IS_NOT_IDENTICAL ||
next_opline->opcode == ZEND_CASE_STRICT) {
if (next_opline->op1_type == IS_CONST
&& Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op1)) == IS_LONG
&& Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op1)) == 0
&& next_opline->op2_type == opline->result_type
&& next_opline->op2.var == opline->result.var) {
must_set_cflags = 1;
} else if (next_opline->op2_type == IS_CONST
&& Z_TYPE_P(RT_CONSTANT(next_opline, next_opline->op2)) == IS_LONG
&& Z_LVAL_P(RT_CONSTANT(next_opline, next_opline->op2)) == 0
&& next_opline->op2_type == opline->result_type
&& next_opline->op2.var == opline->result.var) {
must_set_cflags = 1;
}
}
}
if (opcode == ZEND_MUL &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
Z_LVAL_P(Z_ZV(op2_addr)) == 2) {
if (Z_MODE(op1_addr) == IS_REG && !must_set_cflags) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op1_addr
| add Ra(result_reg), Ra(result_reg)
}
} else if (opcode == ZEND_MUL &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
!must_set_cflags &&
Z_LVAL_P(Z_ZV(op2_addr)) > 0 &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr)))) {
| GET_ZVAL_LVAL result_reg, op1_addr
| shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
} else if (opcode == ZEND_MUL &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
Z_LVAL_P(Z_ZV(op1_addr)) == 2) {
if (Z_MODE(op2_addr) == IS_REG && !must_set_cflags) {
| lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Ra(Z_REG(op2_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op2_addr
| add Ra(result_reg), Ra(result_reg)
}
} else if (opcode == ZEND_MUL &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
!must_set_cflags &&
Z_LVAL_P(Z_ZV(op1_addr)) > 0 &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op1_addr)))) {
| GET_ZVAL_LVAL result_reg, op2_addr
| shl Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op1_addr)))
} else if (opcode == ZEND_DIV &&
(Z_MODE(op2_addr) == IS_CONST_ZVAL &&
zend_long_is_power_of_two(Z_LVAL_P(Z_ZV(op2_addr))))) {
| GET_ZVAL_LVAL result_reg, op1_addr
| shr Ra(result_reg), zend_long_floor_log2(Z_LVAL_P(Z_ZV(op2_addr)))
} else if (opcode == ZEND_ADD &&
!must_set_cflags &&
Z_MODE(op1_addr) == IS_REG &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op2_addr)))) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Z_LVAL_P(Z_ZV(op2_addr))]
} else if (opcode == ZEND_ADD &&
!must_set_cflags &&
Z_MODE(op2_addr) == IS_REG &&
Z_MODE(op1_addr) == IS_CONST_ZVAL &&
IS_SIGNED_32BIT(Z_LVAL_P(Z_ZV(op1_addr)))) {
| lea Ra(result_reg), [Ra(Z_REG(op2_addr))+Z_LVAL_P(Z_ZV(op1_addr))]
} else if (opcode == ZEND_SUB &&
!must_set_cflags &&
Z_MODE(op1_addr) == IS_REG &&
Z_MODE(op2_addr) == IS_CONST_ZVAL &&
IS_SIGNED_32BIT(-Z_LVAL_P(Z_ZV(op2_addr)))) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))-Z_LVAL_P(Z_ZV(op2_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op1_addr
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
may_overflow = 0;
} else if (same_ops && opcode != ZEND_DIV) {
| LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg)
} else {
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_REG(res_addr) != ZREG_R0 && result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else if (Z_REG(res_addr) != ZREG_R1 && result_reg != ZREG_R1) {
tmp_reg = ZREG_R1;
} else {
tmp_reg = ZREG_R2;
}
} else if (result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
tmp_reg = ZREG_R1;
}
| LONG_MATH opcode, result_reg, op2_addr, tmp_reg
(void)tmp_reg;
}
}
if (may_overflow) {
if (res_info & MAY_BE_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) {
| jo &exit_addr
if (Z_MODE(res_addr) == IS_REG && result_reg != Z_REG(res_addr)) {
| mov Ra(Z_REG(res_addr)), Ra(result_reg)
}
} else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| jno &exit_addr
} else {
ZEND_UNREACHABLE();
}
} else {
if (res_info & MAY_BE_LONG) {
| jo >1
} else {
| jno >1
}
}
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && (res_info & MAY_BE_LONG)) {
| SET_ZVAL_LVAL res_addr, Ra(result_reg)
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
}
if (may_overflow && (!(res_info & MAY_BE_GUARD) || (res_info & MAY_BE_ANY) == MAY_BE_DOUBLE)) {
zend_reg tmp_reg1 = ZREG_XMM0;
zend_reg tmp_reg2 = ZREG_XMM1;
if (res_info & MAY_BE_LONG) {
|.cold_code
|1:
}
do {
if ((sizeof(void*) == 8 || Z_MODE(res_addr) != IS_REG) &&
((Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 1) ||
(Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1))) {
if (opcode == ZEND_ADD) {
|.if X64
| mov64 Ra(tmp_reg), 0x43e0000000000000
if (Z_MODE(res_addr) == IS_REG) {
| movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg)
} else {
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg)
}
|.else
| SET_ZVAL_LVAL res_addr, 0
| SET_ZVAL_W2 res_addr, 0x41e00000
|.endif
break;
} else if (opcode == ZEND_SUB) {
|.if X64
| mov64 Ra(tmp_reg), 0xc3e0000000000000
if (Z_MODE(res_addr) == IS_REG) {
| movd xmm(Z_REG(res_addr)-ZREG_XMM0), Ra(tmp_reg)
} else {
| SET_ZVAL_LVAL res_addr, Ra(tmp_reg)
}
|.else
| SET_ZVAL_LVAL res_addr, 0x00200000
| SET_ZVAL_W2 res_addr, 0xc1e00000
|.endif
break;
}
}
| DOUBLE_GET_ZVAL_LVAL tmp_reg1, op1_addr, tmp_reg
| DOUBLE_GET_ZVAL_LVAL tmp_reg2, op2_addr, tmp_reg
if (CAN_USE_AVX()) {
| AVX_MATH_REG opcode, tmp_reg1, tmp_reg1, tmp_reg2
} else {
| SSE_MATH_REG opcode, tmp_reg1, tmp_reg2
}
| DOUBLE_SET_ZVAL_DVAL res_addr, tmp_reg1
} while (0);
if (Z_MODE(res_addr) == IS_MEM_ZVAL
&& (res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
if (res_info & MAY_BE_LONG) {
| jmp >2
|.code
}
|2:
}
return 1;
}
static int zend_jit_math_long_double(dasm_State **Dst,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
zend_reg result_reg =
(Z_MODE(res_addr) == IS_REG) ? Z_REG(res_addr) : ZREG_XMM0;
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
tmp_reg = ZREG_R1;
} else {
tmp_reg = ZREG_R0;
}
| DOUBLE_GET_ZVAL_LVAL result_reg, op1_addr, tmp_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op2_addr, r1
} else {
| SSE_MATH opcode, result_reg, op2_addr, r1
}
} else {
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op2_addr, r0
} else {
| SSE_MATH opcode, result_reg, op2_addr, r0
}
}
| DOUBLE_SET_ZVAL_DVAL res_addr, result_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
return 1;
}
static int zend_jit_math_double_long(dasm_State **Dst,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
zend_reg result_reg, tmp_reg_gp;
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
tmp_reg_gp = ZREG_R1;
} else {
tmp_reg_gp = ZREG_R0;
}
if (zend_is_commutative(opcode)
&& (Z_MODE(res_addr) != IS_REG || Z_MODE(op1_addr) != IS_REG || Z_REG(res_addr) != Z_REG(op1_addr))) {
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
} else {
result_reg = ZREG_XMM0;
}
| DOUBLE_GET_ZVAL_LVAL result_reg, op2_addr, tmp_reg_gp
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op1_addr, r1
} else {
| SSE_MATH opcode, result_reg, op1_addr, r1
}
} else {
if (CAN_USE_AVX()) {
| AVX_MATH opcode, result_reg, result_reg, op1_addr, r0
} else {
| SSE_MATH opcode, result_reg, op1_addr, r0
}
}
} else {
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
tmp_reg = (result_reg == ZREG_XMM0) ? ZREG_XMM1 : ZREG_XMM0;
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
tmp_reg = ZREG_XMM0;
} else {
result_reg = ZREG_XMM0;
tmp_reg = ZREG_XMM1;
}
if (CAN_USE_AVX()) {
zend_reg op1_reg;
if (Z_MODE(op1_addr) == IS_REG) {
op1_reg = Z_REG(op1_addr);
} else {
| DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr
op1_reg = result_reg;
}
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
} else {
| DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp
| AVX_MATH_REG opcode, result_reg, op1_reg, tmp_reg
}
} else {
| DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr
if ((opcode == ZEND_ADD || opcode == ZEND_SUB)
&& Z_MODE(op2_addr) == IS_CONST_ZVAL
&& Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
/* +/- 0 */
} else {
| DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, tmp_reg_gp
| SSE_MATH_REG opcode, result_reg, tmp_reg
}
}
}
| DOUBLE_SET_ZVAL_DVAL res_addr, result_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
}
return 1;
}
static int zend_jit_math_double_double(dasm_State **Dst,
uint8_t opcode,
zend_jit_addr op1_addr,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint32_t res_use_info)
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
if (Z_MODE(res_addr) == IS_REG) {
result_reg = Z_REG(res_addr);
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
} else if (zend_is_commutative(opcode) && Z_MODE(op2_addr) == IS_REG && Z_LAST_USE(op2_addr)) {
result_reg = Z_REG(op2_addr);
} else {
result_reg = ZREG_XMM0;
}
if (CAN_USE_AVX()) {
zend_reg op1_reg;
zend_jit_addr val_addr;
if (Z_MODE(op1_addr) == IS_REG) {
op1_reg = Z_REG(op1_addr);
val_addr = op2_addr;
} else if (Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) {
op1_reg = Z_REG(op2_addr);
val_addr = op1_addr;
} else {
| DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr
op1_reg = result_reg;
val_addr = op2_addr;
}
if ((opcode == ZEND_MUL) &&
Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
| AVX_MATH_REG ZEND_ADD, result_reg, op1_reg, op1_reg
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
| AVX_MATH opcode, result_reg, op1_reg, val_addr, r1
} else {
| AVX_MATH opcode, result_reg, op1_reg, val_addr, r0
}
} else {
zend_jit_addr val_addr;
if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op2_addr) == IS_REG && zend_is_commutative(opcode)) {
| DOUBLE_GET_ZVAL_DVAL result_reg, op2_addr
val_addr = op1_addr;
} else {
| DOUBLE_GET_ZVAL_DVAL result_reg, op1_addr
val_addr = op2_addr;
}
if (same_ops) {
| SSE_MATH_REG opcode, result_reg, result_reg
} else if ((opcode == ZEND_MUL) &&
Z_MODE(val_addr) == IS_CONST_ZVAL && Z_DVAL_P(Z_ZV(val_addr)) == 2.0) {
| SSE_MATH_REG ZEND_ADD, result_reg, result_reg
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
/* ASSIGN_DIM_OP */
| SSE_MATH opcode, result_reg, val_addr, r1
} else {
| SSE_MATH opcode, result_reg, val_addr, r0
}
}
| DOUBLE_SET_ZVAL_DVAL res_addr, result_reg
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) {
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
}
}
return 1;
}
static int zend_jit_math_helper(dasm_State **Dst,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_overflow,
int may_throw)
/* Labels: 1,2,3,4,5,6 */
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
}
}
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) {
if (op2_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >1
|.cold_code
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
}
if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| jmp >5
|.code
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
}
if (!zend_jit_math_long_long(Dst, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
|3:
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops) {
| IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >1
} else {
| IF_NOT_ZVAL_TYPE, op2_addr, IS_DOUBLE, >6
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| jmp >5
}
if (!same_ops) {
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
| jmp >5
}
|.code
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops && (op2_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >1
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & MAY_BE_DOUBLE) {
|.cold_code
}
|1:
if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
if (!zend_jit_math_double_long(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
if (op2_info & MAY_BE_DOUBLE) {
| jmp >5
|.code
}
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & MAY_BE_DOUBLE)) {
if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >6
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) {
if (!same_ops && (op1_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >1
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >6
}
}
if (!zend_jit_math_double_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
}
|1:
if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
}
if (!zend_jit_math_long_double(Dst, opcode, op1_addr, op2_addr, res_addr, res_use_info)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
| jmp >5
|.code
}
}
}
|5:
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
|.cold_code
}
|6:
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1a, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1a, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
| SET_EX_OPLINE opline, r0
if (opcode == ZEND_ADD) {
| EXT_CALL add_function, r0
} else if (opcode == ZEND_SUB) {
| EXT_CALL sub_function, r0
} else if (opcode == ZEND_MUL) {
| EXT_CALL mul_function, r0
} else if (opcode == ZEND_DIV) {
| EXT_CALL div_function, r0
} else {
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
| FREE_OP op1_type, op1, op1_info, 0, NULL
| FREE_OP op2_type, op2, op2_info, 0, NULL
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| jne ->exception_handler_free_op2
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
return 0;
}
}
if ((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
| jmp <5
|.code
}
}
return 1;
}
static int zend_jit_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)));
if (!zend_jit_math_helper(Dst, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_add_arrays(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr)
{
if (Z_MODE(op2_addr) != IS_MEM_ZVAL || Z_REG(op2_addr) != ZREG_FCARG1) {
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
} else if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG2) {
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr
} else {
| GET_ZVAL_LVAL ZREG_R0, op2_addr
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr
| mov FCARG2a, r0
}
| EXT_CALL zend_jit_add_arrays_helper, r0
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_ARRAY_EX
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
return 1;
}
static int zend_jit_long_math_helper(dasm_State **Dst,
const zend_op *opline,
uint8_t opcode,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
zend_ssa_range *op1_range,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
uint32_t res_var,
zend_jit_addr res_addr,
uint32_t res_info,
uint32_t res_use_info,
int may_throw)
/* Labels: 6 */
{
bool same_ops = zend_jit_same_addr(op1_addr, op2_addr);
zend_reg result_reg;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >6
}
if (opcode == ZEND_MOD) {
result_reg = ZREG_RAX;
} else if (Z_MODE(res_addr) == IS_REG) {
if ((opline->opcode == ZEND_SL || opline->opcode == ZEND_SR)
&& opline->op2_type != IS_CONST) {
result_reg = ZREG_R0;
} else {
result_reg = Z_REG(res_addr);
}
} else if (Z_MODE(op1_addr) == IS_REG && Z_LAST_USE(op1_addr)) {
result_reg = Z_REG(op1_addr);
} else if (Z_REG(res_addr) != ZREG_R0) {
result_reg = ZREG_R0;
} else {
/* ASSIGN_DIM_OP */
if (ZREG_FCARG1 == ZREG_RCX
&& (opcode == ZEND_SL || opcode == ZEND_SR)
&& Z_MODE(op2_addr) != IS_CONST_ZVAL) {
result_reg = ZREG_R2;
} else {
result_reg = ZREG_FCARG1;
}
}
if (opcode == ZEND_SL) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
| xor Ra(result_reg), Ra(result_reg)
} else {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
}
} else if (Z_MODE(op1_addr) == IS_REG && op2_lval == 1) {
| lea Ra(result_reg), [Ra(Z_REG(op1_addr))+Ra(Z_REG(op1_addr))]
} else {
| GET_ZVAL_LVAL result_reg, op1_addr
| shl Ra(result_reg), op2_lval
}
} else {
if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) {
| GET_ZVAL_LVAL ZREG_RCX, op2_addr
}
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
| cmp r1, (SIZEOF_ZEND_LONG*8)
| jae >1
|.cold_code
|1:
| cmp r1, 0
| mov Ra(result_reg), 0
| jg >1
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
|.code
}
| GET_ZVAL_LVAL result_reg, op1_addr
| shl Ra(result_reg), cl
|1:
}
} else if (opcode == ZEND_SR) {
| GET_ZVAL_LVAL result_reg, op1_addr
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) {
if (EXPECTED(op2_lval > 0)) {
| sar Ra(result_reg), (SIZEOF_ZEND_LONG * 8) - 1
} else {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
}
} else {
| sar Ra(result_reg), op2_lval
}
} else {
if (Z_MODE(op2_addr) != IS_REG || Z_REG(op2_addr) != ZREG_RCX) {
| GET_ZVAL_LVAL ZREG_RCX, op2_addr
}
if (!op2_range ||
op2_range->min < 0 ||
op2_range->max >= SIZEOF_ZEND_LONG * 8) {
| cmp r1, (SIZEOF_ZEND_LONG*8)
| jae >1
|.cold_code
|1:
| cmp r1, 0
| mov r1, (SIZEOF_ZEND_LONG * 8) - 1
| jg >1
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->negative_shift
|.code
}
|1:
| sar Ra(result_reg), cl
}
} else if (opcode == ZEND_MOD) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr));
if (op2_lval == 0) {
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->mod_by_zero
} else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) {
zval tmp;
zend_jit_addr tmp_addr;
zend_reg tmp_reg;
/* Optimisation for mod of power of 2 */
ZVAL_LONG(&tmp, op2_lval - 1);
tmp_addr = ZEND_ADDR_CONST_ZVAL(&tmp);
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R0) {
tmp_reg = ZREG_R1;
} else if (result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
tmp_reg = ZREG_R1;
}
| GET_ZVAL_LVAL result_reg, op1_addr
| LONG_MATH ZEND_BW_AND, result_reg, tmp_addr, tmp_reg
(void)tmp_reg;
} else {
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov aword T1, r0 // save
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) {
| mov aword T1, Ra(ZREG_RCX) // save
}
result_reg = ZREG_RDX;
if (op2_lval == -1) {
| xor Ra(result_reg), Ra(result_reg)
} else {
| GET_ZVAL_LVAL ZREG_RAX, op1_addr
| GET_ZVAL_LVAL ZREG_RCX, op2_addr
|.if X64
| cqo
|.else
| cdq
|.endif
| idiv Ra(ZREG_RCX)
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov r0, aword T1 // restore
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RCX) {
| mov Ra(ZREG_RCX), aword T1 // restore
}
}
} else {
if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) {
if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
| cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], 0
} else if (Z_MODE(op2_addr) == IS_REG) {
| test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr))
}
| jz >1
|.cold_code
|1:
zend_jit_invalidate_var_if_necessary(Dst, op1_type, op1_addr, op1);
zend_jit_invalidate_var_if_necessary(Dst, op2_type, op2_addr, op2);
| SET_EX_OPLINE opline, r0
| jmp ->mod_by_zero
|.code
}
/* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */
if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) {
if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
| cmp aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)], -1
} else if (Z_MODE(op2_addr) == IS_REG) {
| cmp Ra(Z_REG(op2_addr)), -1
}
| jz >1
|.cold_code
|1:
| SET_ZVAL_LVAL res_addr, 0
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
}
| jmp >5
|.code
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov aword T1, r0 // save
}
result_reg = ZREG_RDX;
| GET_ZVAL_LVAL ZREG_RAX, op1_addr
|.if X64
| cqo
|.else
| cdq
|.endif
if (Z_MODE(op2_addr) == IS_MEM_ZVAL) {
| idiv aword [Ra(Z_REG(op2_addr))+Z_OFFSET(op2_addr)]
} else if (Z_MODE(op2_addr) == IS_REG) {
| idiv Ra(Z_REG(op2_addr))
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RAX) {
| mov r0, aword T1 // restore
}
}
} else if (same_ops) {
| GET_ZVAL_LVAL result_reg, op1_addr
| LONG_MATH_REG opcode, Ra(result_reg), Ra(result_reg)
} else {
zend_reg tmp_reg;
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_REG(res_addr) != ZREG_R0 && result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else if (Z_REG(res_addr) != ZREG_R1 && result_reg != ZREG_R1) {
tmp_reg = ZREG_R1;
} else {
tmp_reg = ZREG_R2;
}
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_R1) {
tmp_reg = ZREG_R0;
} else if (result_reg != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
tmp_reg = ZREG_R1;
}
| GET_ZVAL_LVAL result_reg, op1_addr
| LONG_MATH opcode, result_reg, op2_addr, tmp_reg
(void)tmp_reg;
}
if (Z_MODE(res_addr) != IS_REG || Z_REG(res_addr) != result_reg) {
| SET_ZVAL_LVAL res_addr, Ra(result_reg)
}
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != Z_REG(res_addr) || Z_OFFSET(op1_addr) != Z_OFFSET(res_addr)) {
if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
}
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if ((op1_info & MAY_BE_LONG) &&
(op2_info & MAY_BE_LONG)) {
|.cold_code
}
|6:
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1a, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
} else {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
| LOAD_ZVAL_ADDR FCARG1a, real_addr
} else if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
| SET_EX_OPLINE opline, r0
if (opcode == ZEND_BW_OR) {
| EXT_CALL bitwise_or_function, r0
} else if (opcode == ZEND_BW_AND) {
| EXT_CALL bitwise_and_function, r0
} else if (opcode == ZEND_BW_XOR) {
| EXT_CALL bitwise_xor_function, r0
} else if (opcode == ZEND_SL) {
| EXT_CALL shift_left_function, r0
} else if (opcode == ZEND_SR) {
| EXT_CALL shift_right_function, r0
} else if (opcode == ZEND_MOD) {
| EXT_CALL mod_function, r0
} else {
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) {
/* compound assignment may decrement "op2" refcount */
op2_info |= MAY_BE_RC1;
}
| FREE_OP op1_type, op1, op1_info, 0, NULL
| FREE_OP op2_type, op2, op2_info, 0, NULL
if (may_throw) {
if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| jne ->exception_handler_free_op2
} else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
if (Z_MODE(res_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var);
if (!zend_jit_load_reg(Dst, real_addr, res_addr, res_info)) {
return 0;
}
}
if ((op1_info & MAY_BE_LONG) &&
(op2_info & MAY_BE_LONG)) {
| jmp >5
|.code
}
}
|5:
return 1;
}
static int zend_jit_long_math(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG));
if (!zend_jit_long_math_helper(Dst, opline, opline->opcode,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->result.var, res_addr, res_info, res_use_info, may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
return 1;
}
static int zend_jit_concat_helper(dasm_State **Dst,
const zend_op *opline,
uint8_t op1_type,
znode_op op1,
zend_jit_addr op1_addr,
uint32_t op1_info,
uint8_t op2_type,
znode_op op2,
zend_jit_addr op2_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
int may_throw)
{
#if 1
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
}
if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >6
}
if (Z_MODE(op1_addr) == IS_MEM_ZVAL && Z_REG(op1_addr) == Z_REG(res_addr) && Z_OFFSET(op1_addr) == Z_OFFSET(res_addr)) {
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| EXT_CALL zend_jit_fast_assign_concat_helper, r0
/* concatenation with itself may reduce refcount */
op2_info |= MAY_BE_RC1;
} else {
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
if (op1_type == IS_CV || op1_type == IS_CONST) {
| EXT_CALL zend_jit_fast_concat_helper, r0
} else {
| EXT_CALL zend_jit_fast_concat_tmp_helper, r0
}
|.if not(X64)
| add r4, 12
|.endif
}
/* concatenation with empty string may increase refcount */
op2_info |= MAY_BE_RCN;
| FREE_OP op2_type, op2, op2_info, 0, opline
|5:
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) ||
(op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) {
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
|.cold_code
|6:
}
#endif
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1) {
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
} else {
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, op2_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR op2_addr, r0
|.endif
| SET_EX_OPLINE opline, r0
| EXT_CALL concat_function, r0
|.if not(X64)
| add r4, 12
|.endif
/* concatenation with empty string may increase refcount */
op1_info |= MAY_BE_RCN;
op2_info |= MAY_BE_RCN;
| FREE_OP op1_type, op1, op1_info, 0, NULL
| FREE_OP op2_type, op2, op2_info, 0, NULL
if (may_throw) {
if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) {
zend_jit_check_exception_undef_result(Dst, opline);
} else {
zend_jit_check_exception(Dst);
}
}
#if 1
if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) {
| jmp <5
|.code
}
}
#endif
return 1;
}
static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw);
}
static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr)
/* Labels: 1,2,3,4,5 */
{
zend_jit_addr op2_addr = OP2_ADDR();
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& type == BP_VAR_R
&& !exit_addr) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (op2_info & MAY_BE_LONG) {
bool op2_loaded = 0;
bool packed_loaded = 0;
bool bad_packed_key = 0;
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) {
| // if (EXPECTED(Z_TYPE_P(dim) == IS_LONG))
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3
}
if (op1_info & MAY_BE_PACKED_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jz &exit_addr
} else {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jnz &exit_addr
}
}
if (type == BP_VAR_W) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
op2_loaded = 1;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
zend_long val = -1;
if (Z_MODE(op2_addr) == IS_CONST_ZVAL) {
val = Z_LVAL_P(Z_ZV(op2_addr));
if (val >= 0 && val < HT_MAX_SIZE) {
packed_loaded = 1;
} else {
bad_packed_key = 1;
}
} else {
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
op2_loaded = 1;
}
packed_loaded = 1;
}
if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) {
/* don't generate "fast" code for packed array */
packed_loaded = 0;
}
if (packed_loaded) {
| // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef);
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jz >4 // HASH_FIND
}
| // if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed))
|.if X64
| mov eax, dword [FCARG1a + offsetof(zend_array, nNumUsed)]
if (val == 0) {
| test r0, r0
} else if (val > 0 && !op2_loaded) {
| cmp r0, val
} else {
| cmp r0, FCARG2a
}
|.else
if (val >= 0 && !op2_loaded) {
| cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], val
} else {
| cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], FCARG2a
}
|.endif
if (type == BP_JIT_IS) {
if (not_found_exit_addr) {
| jbe &not_found_exit_addr
} else {
| jbe >9 // NOT_FOUND
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jbe &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jbe &not_found_exit_addr
} else if (type == BP_VAR_RW && not_found_exit_addr) {
| jbe &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jbe >7 // NOT_FOUND
} else {
| jbe >2 // NOT_FOUND
}
| // _ret = &_ht->arPacked[h];
if (val >= 0) {
| mov r0, aword [FCARG1a + offsetof(zend_array, arPacked)]
if (val != 0) {
| add r0, val * sizeof(zval)
}
} else {
|.if X64
| mov r0, FCARG2a
| shl r0, 4
|.else
| imul r0, FCARG2a, sizeof(zval)
|.endif
| add r0, aword [FCARG1a + offsetof(zend_array, arPacked)]
}
}
}
switch (type) {
case BP_JIT_IS:
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
if (packed_loaded) {
| jmp >5
}
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
}
if (packed_loaded) {
| EXT_CALL _zend_hash_index_find, r0
} else {
| EXT_CALL zend_hash_index_find, r0
}
| test r0, r0
if (not_found_exit_addr) {
| jz &not_found_exit_addr
} else {
| jz >9 // NOT_FOUND
}
if (op2_info & MAY_BE_STRING) {
| jmp >5
}
} else if (packed_loaded) {
if (op2_info & MAY_BE_STRING) {
| jmp >5
}
} else if (not_found_exit_addr) {
| jmp &not_found_exit_addr
} else {
| jmp >9 // NOT_FOUND
}
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (packed_loaded) {
if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) {
| IF_NOT_Z_TYPE r0, IS_UNDEF, >8
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
if (!found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
| IF_Z_TYPE r0, IS_UNDEF, &exit_addr
}
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| IF_Z_TYPE r0, IS_UNDEF, &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| IF_Z_TYPE r0, IS_UNDEF, >7 // NOT_FOUND
} else {
| IF_Z_TYPE r0, IS_UNDEF, >2 // NOT_FOUND
}
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (packed_loaded && (op1_info & MAY_BE_ARRAY_NUMERIC_HASH))) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jmp &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jmp &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jmp >7 // NOT_FOUND
} else {
| jmp >2 // NOT_FOUND
}
}
if (!packed_loaded || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
}
if (packed_loaded) {
| EXT_CALL _zend_hash_index_find, r0
} else {
| EXT_CALL zend_hash_index_find, r0
}
| test r0, r0
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jz &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jz &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jz >7 // NOT_FOUND
} else {
| jz >2 // NOT_FOUND
}
}
|.cold_code
|2:
switch (type) {
case BP_VAR_R:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
| // zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval);
| // retval = &EG(uninitialized_zval);
| UNDEFINED_OFFSET opline
| jmp >9
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
if (!not_found_exit_addr && !found_exit_addr) {
| // retval = &EG(uninitialized_zval);
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >9
}
break;
default:
ZEND_UNREACHABLE();
}
|.code
break;
case BP_VAR_RW:
if (packed_loaded && !not_found_exit_addr) {
| IF_NOT_Z_TYPE r0, IS_UNDEF, >8
}
if (!packed_loaded ||
!not_found_exit_addr ||
(op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) {
if (packed_loaded && not_found_exit_addr) {
|.cold_code
}
|2:
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
}
if (packed_loaded) {
| EXT_CALL zend_jit_hash_index_lookup_rw_no_packed, r0
} else {
| EXT_CALL zend_jit_hash_index_lookup_rw, r0
}
| test r0, r0
if (not_found_exit_addr) {
if (packed_loaded) {
| jnz >8
| jmp &not_found_exit_addr
|.code
} else {
| jz &not_found_exit_addr
}
} else {
| jz >9
}
}
break;
case BP_VAR_W:
if (packed_loaded) {
| IF_NOT_Z_TYPE r0, IS_UNDEF, >8
}
if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) {
|2:
|4:
if (!op2_loaded) {
| // hval = Z_LVAL_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
}
| EXT_CALL zend_hash_index_lookup, r0
}
break;
default:
ZEND_UNREACHABLE();
}
if (type != BP_JIT_IS && (op2_info & MAY_BE_STRING)) {
| jmp >8
}
}
if (op2_info & MAY_BE_STRING) {
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
| // if (EXPECTED(Z_TYPE_P(dim) == IS_STRING))
| IF_NOT_ZVAL_TYPE op2_addr, IS_STRING, >3
}
| // offset_key = Z_STR_P(dim);
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
| // retval = zend_hash_find(ht, offset_key);
switch (type) {
case BP_JIT_IS:
if (opline->op2_type != IS_CONST) {
| cmp byte [FCARG2a + offsetof(zend_string, val)], '9'
| jle >1
|.cold_code
|1:
| EXT_CALL zend_jit_symtable_find, r0
| jmp >1
|.code
| EXT_CALL zend_hash_find, r0
|1:
} else {
| EXT_CALL zend_hash_find_known_hash, r0
}
| test r0, r0
if (not_found_exit_addr) {
| jz &not_found_exit_addr
} else {
| jz >9 // NOT_FOUND
}
break;
case BP_VAR_R:
case BP_VAR_IS:
case BP_VAR_UNSET:
if (opline->op2_type != IS_CONST) {
| cmp byte [FCARG2a + offsetof(zend_string, val)], '9'
| jle >1
|.cold_code
|1:
| EXT_CALL zend_jit_symtable_find, r0
| jmp >1
|.code
| EXT_CALL zend_hash_find, r0
|1:
} else {
| EXT_CALL zend_hash_find_known_hash, r0
}
| test r0, r0
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) {
| jz &exit_addr
} else if (type == BP_VAR_IS && not_found_exit_addr) {
| jz &not_found_exit_addr
} else if (type == BP_VAR_IS && found_exit_addr) {
| jz >7 // NOT_FOUND
} else {
| jz >2 // NOT_FOUND
|.cold_code
|2:
switch (type) {
case BP_VAR_R:
// zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key));
| UNDEFINED_INDEX opline
| jmp >9
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
| // retval = &EG(uninitialized_zval);
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >9
break;
default:
ZEND_UNREACHABLE();
}
|.code
}
break;
case BP_VAR_RW:
if (opline->op2_type != IS_CONST) {
| EXT_CALL zend_jit_symtable_lookup_rw, r0
} else {
| EXT_CALL zend_jit_hash_lookup_rw, r0
}
| test r0, r0
if (not_found_exit_addr) {
| jz &not_found_exit_addr
} else {
| jz >9
}
break;
case BP_VAR_W:
if (opline->op2_type != IS_CONST) {
| EXT_CALL zend_jit_symtable_lookup_w, r0
} else {
| EXT_CALL zend_hash_lookup, r0
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (type == BP_JIT_IS && (op2_info & (MAY_BE_LONG|MAY_BE_STRING))) {
|5:
if (op1_info & MAY_BE_ARRAY_OF_REF) {
| ZVAL_DEREF r0, MAY_BE_REF
}
| cmp byte [r0 + 8], IS_NULL
if (not_found_exit_addr) {
| jle &not_found_exit_addr
} else if (found_exit_addr) {
| jg &found_exit_addr
} else {
| jle >9 // NOT FOUND
}
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
|.cold_code
|3:
}
if (type != BP_VAR_RW) {
| SET_EX_OPLINE opline, r0
}
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
switch (type) {
case BP_VAR_R:
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_fetch_dim_r_helper, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
break;
case BP_JIT_IS:
| EXT_CALL zend_jit_fetch_dim_isset_helper, r0
| test r0, r0
if (not_found_exit_addr) {
| je &not_found_exit_addr
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
| jmp >8
}
} else if (found_exit_addr) {
| jne &found_exit_addr
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
| jmp >9
}
} else {
| jne >8
| jmp >9
}
break;
case BP_VAR_IS:
case BP_VAR_UNSET:
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_fetch_dim_is_helper, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
break;
case BP_VAR_RW:
| EXT_CALL zend_jit_fetch_dim_rw_helper, r0
| test r0, r0
| jne >8
| jmp >9
break;
case BP_VAR_W:
| EXT_CALL zend_jit_fetch_dim_w_helper, r0
| test r0, r0
| jne >8
| jmp >9
break;
default:
ZEND_UNREACHABLE();
}
if (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) {
|.code
}
}
return 1;
}
static int zend_jit_simple_assign(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
int in_cold,
int save_r1,
bool check_exception)
/* Labels: 1,2,3 */
{
zend_reg tmp_reg;
if (Z_MODE(var_addr) == IS_REG || Z_REG(var_addr) != ZREG_R0) {
tmp_reg = ZREG_R0;
} else {
/* ASSIGN_DIM */
tmp_reg = ZREG_FCARG1;
}
if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
zval *zv = Z_ZV(val_addr);
if (!res_addr) {
| ZVAL_COPY_CONST var_addr, var_info, var_def_info, zv, tmp_reg
} else {
| ZVAL_COPY_CONST_2 var_addr, res_addr, var_info, var_def_info, zv, tmp_reg
}
if (Z_REFCOUNTED_P(zv)) {
if (!res_addr) {
| ADDREF_CONST zv, Ra(tmp_reg)
} else {
| ADDREF_CONST_2 zv, Ra(tmp_reg)
}
}
} else {
if (val_info & MAY_BE_UNDEF) {
if (in_cold) {
| IF_NOT_ZVAL_TYPE val_addr, IS_UNDEF, >2
} else {
| IF_ZVAL_TYPE val_addr, IS_UNDEF, >1
|.cold_code
|1:
}
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
if (save_r1) {
| mov aword T1, FCARG1a // save
}
| SET_ZVAL_TYPE_INFO var_addr, IS_NULL
if (res_addr) {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
}
if (opline) {
| SET_EX_OPLINE opline, Ra(tmp_reg)
}
ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP);
| mov FCARG1d, Z_OFFSET(val_addr)
| EXT_CALL zend_jit_undefined_op_helper, r0
if (check_exception) {
| test r0, r0
| jz ->exception_handler_undef
}
if (save_r1) {
| mov FCARG1a, aword T1 // restore
}
| jmp >3
if (in_cold) {
|2:
} else {
|.code
}
}
if (val_info & MAY_BE_REF) {
if (val_type == IS_CV) {
ZEND_ASSERT(Z_REG(var_addr) != ZREG_R2);
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_R2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR r2, val_addr
}
| ZVAL_DEREF r2, val_info
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0);
} else {
zend_jit_addr ref_addr;
zend_reg type_reg = tmp_reg;
if (in_cold) {
| IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, >1
} else {
| IF_ZVAL_TYPE val_addr, IS_REFERENCE, >1
|.cold_code
|1:
}
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR r2, val_addr
| GC_DELREF r2
| // ZVAL_COPY_VALUE(return_value, &ref->value);
ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 8);
if (!res_addr) {
| ZVAL_COPY_VALUE var_addr, var_info, ref_addr, val_info, type_reg, tmp_reg
} else {
| ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, ref_addr, val_info, type_reg, tmp_reg
}
| je >2
if (tmp_reg == ZREG_R0) {
| IF_NOT_REFCOUNTED ah, >3
} else {
| IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >3
}
| GET_ZVAL_PTR Ra(tmp_reg), var_addr
if (!res_addr) {
| GC_ADDREF Ra(tmp_reg)
} else {
| add dword [Ra(tmp_reg)], 2
}
| jmp >3
|2:
if (res_addr) {
if (tmp_reg == ZREG_R0) {
| IF_NOT_REFCOUNTED ah, >2
} else {
| IF_NOT_FLAGS Rd(tmp_reg), (IS_TYPE_REFCOUNTED << Z_TYPE_FLAGS_SHIFT), >2
}
| GET_ZVAL_PTR Ra(tmp_reg), var_addr
| GC_ADDREF Ra(tmp_reg)
|2:
}
if (save_r1) {
| mov aword T1, FCARG1a // save
}
| EFREE_REFERENCE r2
if (save_r1) {
| mov FCARG1a, aword T1 // restore
}
| jmp >3
if (in_cold) {
|1:
} else {
|.code
}
}
}
if (!res_addr) {
| ZVAL_COPY_VALUE var_addr, var_info, val_addr, val_info, ZREG_R2, tmp_reg
} else {
| ZVAL_COPY_VALUE_2 var_addr, var_info, res_addr, val_addr, val_info, ZREG_R2, tmp_reg
}
if (val_type == IS_CV) {
if (!res_addr) {
| TRY_ADDREF val_info, dh, Ra(tmp_reg)
} else {
| TRY_ADDREF_2 val_info, dh, Ra(tmp_reg)
}
} else {
if (res_addr) {
| TRY_ADDREF val_info, dh, Ra(tmp_reg)
}
}
|3:
}
return 1;
}
static int zend_jit_assign_to_typed_ref(dasm_State **Dst,
const zend_op *opline,
uint8_t val_type,
zend_jit_addr val_addr,
zend_jit_addr res_addr,
bool check_exception)
{
| // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >2
|.cold_code
|2:
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, r0
}
if (!res_addr) {
if (val_type == IS_CONST) {
| EXT_CALL zend_jit_assign_const_to_typed_ref, r0
} else if (val_type == IS_TMP_VAR) {
| EXT_CALL zend_jit_assign_tmp_to_typed_ref, r0
} else if (val_type == IS_VAR) {
| EXT_CALL zend_jit_assign_var_to_typed_ref, r0
} else if (val_type == IS_CV) {
| EXT_CALL zend_jit_assign_cv_to_typed_ref, r0
} else {
ZEND_UNREACHABLE();
}
} else {
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
if (val_type == IS_CONST) {
| EXT_CALL zend_jit_assign_const_to_typed_ref2, r0
} else if (val_type == IS_TMP_VAR) {
| EXT_CALL zend_jit_assign_tmp_to_typed_ref2, r0
} else if (val_type == IS_VAR) {
| EXT_CALL zend_jit_assign_var_to_typed_ref2, r0
} else if (val_type == IS_CV) {
| EXT_CALL zend_jit_assign_cv_to_typed_ref2, r0
} else {
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
}
if (check_exception) {
| // if (UNEXPECTED(EG(exception) != NULL)) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| je >8 // END OF zend_jit_assign_to_variable()
| jmp ->exception_handler
} else {
| jmp >8
}
|.code
return 1;
}
static int zend_jit_assign_to_variable_call(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr __var_use_addr,
zend_jit_addr var_addr,
uint32_t __var_info,
uint32_t __var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr __res_addr,
bool __check_exception)
{
if (val_info & MAY_BE_UNDEF) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE val_addr, IS_UNDEF, &exit_addr
} else {
| IF_ZVAL_TYPE val_addr, IS_UNDEF, >1
|.cold_code
|1:
ZEND_ASSERT(Z_REG(val_addr) == ZREG_FP);
if (Z_REG(var_addr) != ZREG_FP) {
| mov aword T1, Ra(Z_REG(var_addr)) // save
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, Z_OFFSET(val_addr)
| EXT_CALL zend_jit_undefined_op_helper, r0
if (Z_REG(var_addr) != ZREG_FP) {
| mov Ra(Z_REG(var_addr)), aword T1 // restore
}
if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, var_addr
}
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| call ->assign_const
| jmp >9
|.code
}
}
if (Z_MODE(var_addr) != IS_MEM_ZVAL || Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, var_addr
}
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, r0
}
if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| call ->assign_tmp
} else if (val_type == IS_CONST) {
| call ->assign_const
} else if (val_type == IS_TMP_VAR) {
| call ->assign_tmp
} else if (val_type == IS_VAR) {
if (!(val_info & MAY_BE_REF)) {
| call ->assign_tmp
} else {
| call ->assign_var
}
} else if (val_type == IS_CV) {
if (!(val_info & MAY_BE_REF)) {
| call ->assign_cv_noref
} else {
| call ->assign_cv
}
if ((val_info & MAY_BE_UNDEF) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
|9:
}
} else {
ZEND_UNREACHABLE();
}
return 1;
}
static int zend_jit_assign_to_variable(dasm_State **Dst,
const zend_op *opline,
zend_jit_addr var_use_addr,
zend_jit_addr var_addr,
uint32_t var_info,
uint32_t var_def_info,
uint8_t val_type,
zend_jit_addr val_addr,
uint32_t val_info,
zend_jit_addr res_addr,
bool check_exception)
/* Labels: 1,2,3,4,5,8 */
{
int done = 0;
zend_reg ref_reg, tmp_reg;
if (Z_MODE(var_addr) == IS_REG || Z_REG(var_use_addr) != ZREG_R0) {
ref_reg = ZREG_FCARG1;
tmp_reg = ZREG_R0;
} else {
/* ASSIGN_DIM */
ref_reg = ZREG_R0;
tmp_reg = ZREG_FCARG1;
}
if (var_info & MAY_BE_REF) {
if (Z_MODE(var_use_addr) != IS_MEM_ZVAL || Z_REG(var_use_addr) != ref_reg || Z_OFFSET(var_use_addr) != 0) {
| LOAD_ZVAL_ADDR Ra(ref_reg), var_use_addr
var_addr = var_use_addr = ZEND_ADDR_MEM_ZVAL(ref_reg, 0);
}
| // if (Z_ISREF_P(variable_ptr)) {
| IF_NOT_Z_TYPE, Ra(ref_reg), IS_REFERENCE, >3
| // if (UNEXPECTED(ZEND_REF_HAS_TYPE_SOURCES(Z_REF_P(variable_ptr)))) {
| GET_Z_PTR FCARG1a, Ra(ref_reg)
if (!zend_jit_assign_to_typed_ref(Dst, opline, val_type, val_addr, res_addr, check_exception)) {
return 0;
}
| lea Ra(ref_reg), [FCARG1a + offsetof(zend_reference, val)]
|3:
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (RC_MAY_BE_1(var_info)) {
int in_cold = 0;
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_ZVAL_REFCOUNTED var_use_addr, >1
|.cold_code
|1:
in_cold = 1;
}
if (Z_REG(var_use_addr) == ZREG_FCARG1 || Z_REG(var_use_addr) == ZREG_R0) {
bool keep_gc = 0;
| GET_ZVAL_PTR Ra(tmp_reg), var_use_addr
if (tmp_reg == ZREG_FCARG1) {
if (Z_MODE(val_addr) == IS_REG) {
keep_gc = 1;
} else if ((val_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) == 0) {
keep_gc = 1;
} else if (Z_MODE(val_addr) == IS_CONST_ZVAL) {
if (sizeof(void*) == 4) {
keep_gc = 1;
} else {
zval *zv = Z_ZV(val_addr);
if (Z_TYPE_P(zv) == IS_DOUBLE) {
if (Z_DVAL_P(zv) == 0 || IS_SIGNED_32BIT(zv)) {
keep_gc = 1;
}
} else if (IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
keep_gc = 1;
}
}
} else if (Z_MODE(val_addr) == IS_MEM_ZVAL) {
if ((val_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_GUARD)) == MAY_BE_DOUBLE) {
keep_gc = 1;
}
}
}
if (!keep_gc) {
| mov aword T1, Ra(tmp_reg) // save
}
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 0, 0)) {
return 0;
}
if (!keep_gc) {
| mov FCARG1a, aword T1 // restore
}
} else {
| GET_ZVAL_PTR FCARG1a, var_use_addr
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, in_cold, 1, 0)) {
return 0;
}
}
| GC_DELREF FCARG1a
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
| jnz >4
} else {
| jnz >8
}
| ZVAL_DTOR_FUNC var_info, opline
if (in_cold || (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0)) {
if (check_exception && !(val_info & MAY_BE_UNDEF)) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| je >8
| jmp ->exception_handler
} else {
| jmp >8
}
}
if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) {
|4:
| IF_GC_MAY_NOT_LEAK FCARG1a, >8
if (opline) {
| SET_EX_OPLINE opline, r0
}
| EXT_CALL gc_possible_root, r0
if (in_cold) {
| jmp >8
}
}
if (check_exception && (val_info & MAY_BE_UNDEF)) {
|8:
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| je >8
| jmp ->exception_handler
}
if (in_cold) {
|.code
} else {
done = 1;
}
} else /* if (RC_MAY_BE_N(var_info)) */ {
if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_ZVAL_REFCOUNTED var_use_addr, >5
}
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) {
if (Z_REG(var_use_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(var_use_addr)) // save
}
| GET_ZVAL_PTR FCARG1a, var_use_addr
| GC_DELREF FCARG1a
| IF_GC_MAY_NOT_LEAK FCARG1a, >5
if (opline) {
| SET_EX_OPLINE opline, r0
}
| EXT_CALL gc_possible_root, r0
if (Z_REG(var_use_addr) != ZREG_FP) {
| mov Ra(Z_REG(var_use_addr)), T1 // restore
}
} else {
| GET_ZVAL_PTR Ra(tmp_reg), var_use_addr
| GC_DELREF Ra(tmp_reg)
}
|5:
}
}
if (!done && !zend_jit_simple_assign(Dst, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0, 0, check_exception)) {
return 0;
}
|8:
return 1;
}
static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw)
{
zend_jit_addr op2_addr, op3_addr, res_addr;
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
op3_addr = OP1_DATA_ADDR();
if (opline->result_type == IS_UNUSED) {
res_addr = 0;
} else {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE op3_addr, IS_UNDEF, &exit_addr
val_info &= ~MAY_BE_UNDEF;
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG2a, FCARG1a
| IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2
| lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)]
| jmp >3
|.cold_code
|2:
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_prepare_assign_dim_ref, r0
| test r0, r0
| mov FCARG1a, r0
| jne >1
| jmp ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL
| jg >7
}
| // ZVAL_ARR(container, zend_new_array(8));
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|6:
if (opline->op2_type == IS_UNUSED) {
uint32_t var_info = MAY_BE_NULL;
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, r0
| // if (UNEXPECTED(!var_ptr)) {
| test r0, r0
| jz >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| jmp >9
|.code
if (!zend_jit_simple_assign(Dst, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0, 0)) {
return 0;
}
} else {
uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) {
return 0;
}
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
|8:
| // value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE);
if (opline->op1_type == IS_VAR) {
ZEND_ASSERT(opline->result_type == IS_UNUSED);
if (!zend_jit_assign_to_variable_call(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) {
return 0;
}
}
}
}
if (((op1_info & MAY_BE_ARRAY) &&
(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) ||
(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY)))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) &&
(op1_info & MAY_BE_ARRAY)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL
| jg >2
}
| // ZVAL_ARR(container, zend_new_array(8));
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
| // ZEND_VM_C_GOTO(assign_dim_op_new_array);
| jmp <6
|2:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
|.if not(X64)
| sub r4, 8
|.endif
if (opline->result_type == IS_UNUSED) {
|.if X64
| xor CARG4, CARG4
|.else
| push 0
|.endif
} else {
|.if X64
| LOAD_ZVAL_ADDR CARG4, res_addr
|.else
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, op3_addr
|.else
| PUSH_ZVAL_ADDR op3_addr, r0
|.endif
| EXT_CALL zend_jit_assign_dim_helper, r0
|.if not(X64)
| add r4, 8
|.endif
#ifdef ZEND_JIT_USE_RC_INFERENCE
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) {
/* ASSIGN_DIM may increase refcount of the value */
val_info |= MAY_BE_RCN;
}
#endif
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, NULL
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| jmp >9 // END
}
|.code
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
|9:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (may_throw) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw)
{
zend_jit_addr op2_addr, op3_addr, var_addr;
const void *not_found_exit_addr = NULL;
uint32_t var_info = MAY_BE_NULL;
ZEND_ASSERT(opline->result_type == IS_UNUSED);
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
op3_addr = OP1_DATA_ADDR();
| SET_EX_OPLINE opline, r0
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG2a, FCARG1a
| IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2
| lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)]
| jmp >3
|.cold_code
|2:
| EXT_CALL zend_jit_prepare_assign_dim_ref, r0
| test r0, r0
| mov FCARG1a, r0
| jne >1
| jmp ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL
| jg >7
}
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & MAY_BE_NULL) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
| // ZVAL_ARR(container, zend_new_array(8));
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
if (op1_info & MAY_BE_ARRAY) {
| jmp >1
|.code
|1:
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0);
|6:
if (opline->op2_type == IS_UNUSED) {
var_info = MAY_BE_NULL;
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, r0
| // if (UNEXPECTED(!var_ptr)) {
| test r0, r0
| jz >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| jmp >9
|.code
} else {
var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0);
if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) {
var_info |= MAY_BE_REF;
}
if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
var_info |= MAY_BE_RC1;
}
if (dim_type != IS_UNKNOWN
&& dim_type != IS_UNDEF
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY
&& (op2_info & (MAY_BE_LONG|MAY_BE_STRING))
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, not_found_exit_addr, NULL)) {
return 0;
}
|8:
if (not_found_exit_addr && dim_type != IS_REFERENCE) {
| IF_NOT_Z_TYPE, r0, dim_type, &not_found_exit_addr
var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
| IF_NOT_Z_TYPE, r0, IS_REFERENCE, >1
| GET_Z_PTR FCARG1a, r0
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >2
| lea r0, aword [FCARG1a + offsetof(zend_reference, val)]
|.cold_code
|2:
| LOAD_ZVAL_ADDR FCARG2a, op3_addr
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (op1_data_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
|.code
|1:
}
}
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, may_throw)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
(opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info,
op1_data_range,
0, var_addr, var_def_info, var_info, may_throw)) {
return 0;
}
break;
case ZEND_CONCAT:
if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr,
may_throw)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
binary_op_type binary_op;
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
binary_op = get_binary_op(opline->extended_value);
|.if X64
| LOAD_ZVAL_ADDR CARG3, op3_addr
| LOAD_ADDR CARG4, binary_op
|.else
| sub r4, 8
| PUSH_ADDR binary_op, r0
| PUSH_ZVAL_ADDR op3_addr, r0
|.endif
| EXT_CALL zend_jit_assign_dim_op_helper, r0
|.if not(X64)
| add r4, 8
|.endif
|9:
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, NULL
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL
if (may_throw) {
zend_jit_check_exception(Dst);
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
| jmp >9 // END
|.code
|9:
}
} else if ((op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY))
&& (!not_found_exit_addr || (var_info & MAY_BE_REF))) {
|.cold_code
|9:
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, op1_data_info, 0, opline
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (may_throw) {
zend_jit_check_exception(Dst);
}
| jmp >9
|.code
|9:
}
return 1;
}
static int zend_jit_assign_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_ssa_range *op1_range, uint32_t op2_info, zend_ssa_range *op2_range, int may_overflow, int may_throw)
{
zend_jit_addr op1_addr, op2_addr;
ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF));
op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
if (op1_info & MAY_BE_REF) {
binary_op_type binary_op = get_binary_op(opline->extended_value);
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE, FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG1a, FCARG1a
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >2
| add FCARG1a, offsetof(zend_reference, val)
|.cold_code
|2:
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
| SET_EX_OPLINE opline, r0
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
zend_jit_check_exception(Dst);
| jmp >9
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
int result;
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_DIV:
result = zend_jit_math_helper(Dst, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_addr, op1_def_info, op1_info, may_overflow, may_throw);
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
result = zend_jit_long_math_helper(Dst, opline, opline->extended_value,
opline->op1_type, opline->op1, op1_addr, op1_info, op1_range,
opline->op2_type, opline->op2, op2_addr, op2_info, op2_range,
opline->op1.var, op1_addr, op1_def_info, op1_info, may_throw);
break;
case ZEND_CONCAT:
result = zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_addr, may_throw);
break;
default:
ZEND_UNREACHABLE();
}
|9:
return result;
}
static int zend_jit_cmp_long_long(dasm_State **Dst,
const zend_op *opline,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
bool swap = 0;
bool result;
if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) {
if (!smart_branch_opcode ||
smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
| SET_ZVAL_TYPE_INFO res_addr, (result ? IS_TRUE : IS_FALSE)
}
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
if (!result) {
| jmp => target_label
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
if (result) {
| jmp => target_label
}
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
if (skip_comparison) {
if (Z_MODE(op1_addr) != IS_REG &&
(Z_MODE(op2_addr) == IS_REG ||
(Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL))) {
swap = 1;
}
} else if (Z_MODE(op1_addr) == IS_REG) {
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
| test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr))
} else {
| LONG_OP cmp, Z_REG(op1_addr), op2_addr, r0
}
} else if (Z_MODE(op2_addr) == IS_REG) {
if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op1_addr)) == 0) {
| test Ra(Z_REG(op2_addr)), Ra(Z_REG(op2_addr))
} else {
| LONG_OP cmp, Z_REG(op2_addr), op1_addr, r0
}
swap = 1;
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) != IS_CONST_ZVAL) {
| LONG_OP_WITH_CONST cmp, op2_addr, Z_LVAL_P(Z_ZV(op1_addr))
swap = 1;
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_MODE(op1_addr) != IS_CONST_ZVAL) {
| LONG_OP_WITH_CONST cmp, op1_addr, Z_LVAL_P(Z_ZV(op2_addr))
} else {
| GET_ZVAL_LVAL ZREG_R0, op1_addr
if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 0) {
| test r0, r0
} else {
| LONG_OP cmp, ZREG_R0, op2_addr, r0
}
}
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| setne al
break;
case ZEND_IS_SMALLER:
if (swap) {
| setg al
} else {
| setl al
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setge al
} else {
| setle al
}
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| jne &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| jle &exit_addr
} else {
| jle => target_label
}
} else {
if (exit_addr) {
| jge &exit_addr
} else {
| jge => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jl &exit_addr
} else {
| jl => target_label
}
} else {
if (exit_addr) {
| jg &exit_addr
} else {
| jg => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| je &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| jg &exit_addr
} else {
| jg => target_label
}
} else {
if (exit_addr) {
| jl &exit_addr
} else {
| jl => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jge &exit_addr
} else {
| jge => target_label
}
} else {
if (exit_addr) {
| jle &exit_addr
} else {
| jle => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| setne al
break;
case ZEND_IS_SMALLER:
if (swap) {
| setg al
} else {
| setl al
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setge al
} else {
| setle al
}
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static int zend_jit_cmp_double_common(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, bool swap, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
if (exit_addr) {
| jne &exit_addr
| jp &exit_addr
} else {
| jne => target_label
| jp => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
| jp >1
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
|1:
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| jne &exit_addr
| jp &exit_addr
} else {
| jp >1
| je => target_label
|1:
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| jbe &exit_addr
} else {
| jbe => target_label
}
} else {
if (exit_addr) {
| jae &exit_addr
| jp &exit_addr
} else {
| jae => target_label
| jp => target_label
}
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jb &exit_addr
} else {
| jb => target_label
}
} else {
if (exit_addr) {
| ja &exit_addr
| jp &exit_addr
} else {
| ja => target_label
| jp => target_label
}
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jp >1
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
|1:
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| jne &exit_addr
| jp &exit_addr
} else {
| jne => target_label
| jp => target_label
}
break;
case ZEND_IS_NOT_IDENTICAL:
if (exit_addr) {
| jp >1
| je &exit_addr
|1:
} else {
| jne => target_label
| jp => target_label
}
break;
case ZEND_IS_SMALLER:
if (swap) {
if (exit_addr) {
| ja &exit_addr
} else {
| ja => target_label
}
} else {
| jp >1
if (exit_addr) {
| jb &exit_addr
} else {
| jb => target_label
}
|1:
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
if (exit_addr) {
| jae &exit_addr
} else {
| jae => target_label
}
} else {
| jp >1
if (exit_addr) {
| jbe &exit_addr
} else {
| jbe => target_label
}
|1:
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jne => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| je => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
break;
case ZEND_IS_SMALLER:
if (swap) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jbe => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jae => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jb => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| ja => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| je => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jne => target_label
| jp => target_label
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
break;
case ZEND_IS_SMALLER:
if (swap) {
| seta al
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
| ja => target_label
} else {
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jb => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setae al
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
| jae => target_label
} else {
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jbe => target_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
break;
default:
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_CASE:
case ZEND_CASE_STRICT:
| jp >1
| mov eax, IS_TRUE
| je >2
|1:
| mov eax, IS_FALSE
|2:
break;
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_NOT_IDENTICAL:
| jp >1
| mov eax, IS_FALSE
| je >2
|1:
| mov eax, IS_TRUE
|2:
break;
case ZEND_IS_SMALLER:
if (swap) {
| seta al
| movzx eax, al
| add eax, 2
} else {
| jp >1
| mov eax, IS_TRUE
| jb >2
|1:
| mov eax, IS_FALSE
|2:
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (swap) {
| setae al
| movzx eax, al
| add eax, 2
} else {
| jp >1
| mov eax, IS_TRUE
| jbe >2
|1:
| mov eax, IS_FALSE
|2:
}
break;
default:
ZEND_UNREACHABLE();
}
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static int zend_jit_cmp_long_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_XMM0;
| DOUBLE_GET_ZVAL_LVAL tmp_reg, op1_addr, ZREG_R0
| DOUBLE_CMP tmp_reg, op2_addr
return zend_jit_cmp_double_common(Dst, opline, res_addr, 0, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_double_long(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_reg tmp_reg = ZREG_XMM0;
| DOUBLE_GET_ZVAL_LVAL tmp_reg, op2_addr, ZREG_R0
| DOUBLE_CMP tmp_reg, op1_addr
return zend_jit_cmp_double_common(Dst, opline, res_addr, /* swap */ 1, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_double_double(dasm_State **Dst, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
bool swap = 0;
if (Z_MODE(op1_addr) == IS_REG) {
| DOUBLE_CMP Z_REG(op1_addr), op2_addr
} else if (Z_MODE(op2_addr) == IS_REG) {
| DOUBLE_CMP Z_REG(op2_addr), op1_addr
swap = 1;
} else {
zend_reg tmp_reg = ZREG_XMM0;
| DOUBLE_GET_ZVAL_DVAL tmp_reg, op1_addr
| DOUBLE_CMP tmp_reg, op2_addr
}
return zend_jit_cmp_double_common(Dst, opline, res_addr, swap, smart_branch_opcode, target_label, target_label2, exit_addr);
}
static int zend_jit_cmp_slow(dasm_State **Dst, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
| test, eax, eax
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ_EX ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
| setne al
break;
case ZEND_IS_SMALLER:
| setl al
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| setle al
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (smart_branch_opcode == ZEND_JMPZ ||
smart_branch_opcode == ZEND_JMPZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_SMALLER:
if (exit_addr) {
| jge &exit_addr
} else {
| jge => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (exit_addr) {
| jg &exit_addr
} else {
| jg => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
} else if (smart_branch_opcode == ZEND_JMPNZ ||
smart_branch_opcode == ZEND_JMPNZ_EX) {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
if (exit_addr) {
| je &exit_addr
} else {
| je => target_label
}
break;
case ZEND_IS_NOT_EQUAL:
if (exit_addr) {
| jne &exit_addr
} else {
| jne => target_label
}
break;
case ZEND_IS_SMALLER:
if (exit_addr) {
| jl &exit_addr
} else {
| jl => target_label
}
break;
case ZEND_IS_SMALLER_OR_EQUAL:
if (exit_addr) {
| jle &exit_addr
} else {
| jle => target_label
}
break;
default:
ZEND_UNREACHABLE();
}
} else {
ZEND_UNREACHABLE();
}
} else {
switch (opline->opcode) {
case ZEND_IS_EQUAL:
case ZEND_CASE:
| sete al
break;
case ZEND_IS_NOT_EQUAL:
| setne al
break;
case ZEND_IS_SMALLER:
| setl al
break;
case ZEND_IS_SMALLER_OR_EQUAL:
| setle al
break;
default:
ZEND_UNREACHABLE();
}
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static int zend_jit_cmp(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
bool same_ops = (opline->op1_type == opline->op2_type) && (opline->op1.var == opline->op2.var);
bool has_slow;
has_slow =
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) &&
((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))));
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >4
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9
}
}
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if (op2_info & MAY_BE_DOUBLE) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >3
|.cold_code
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
|.code
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
}
if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
|4:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >5
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
}
if (!same_ops) {
|5:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
| jmp >6
}
|.code
}
} else if ((op1_info & MAY_BE_DOUBLE) &&
!(op1_info & MAY_BE_LONG) &&
(op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
}
if (op2_info & MAY_BE_DOUBLE) {
if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops && (op2_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >3
} else {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (!same_ops && (op2_info & MAY_BE_LONG)) {
if (op2_info & MAY_BE_DOUBLE) {
|.cold_code
}
|3:
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_LONG, >9
}
if (!zend_jit_cmp_double_long(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op2_info & MAY_BE_DOUBLE) {
| jmp >6
|.code
}
}
} else if ((op2_info & MAY_BE_DOUBLE) &&
!(op2_info & MAY_BE_LONG) &&
(op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) {
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_DOUBLE, >9
}
if (op1_info & MAY_BE_DOUBLE) {
if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) {
if (!same_ops && (op1_info & MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_DOUBLE, >9
}
}
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
}
if (!same_ops && (op1_info & MAY_BE_LONG)) {
if (op1_info & MAY_BE_DOUBLE) {
|.cold_code
}
|3:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >9
}
if (!zend_jit_cmp_long_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (op1_info & MAY_BE_DOUBLE) {
| jmp >6
|.code
}
}
}
if (has_slow ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) ||
(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
if (has_slow) {
|.cold_code
|9:
}
| SET_EX_OPLINE opline, r0
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) {
| IF_NOT_Z_TYPE FCARG1a, IS_UNDEF, >1
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
| LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
|1:
}
if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
| mov T1, FCARG1a // save
| mov FCARG1a, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
| mov FCARG1a, T1 // restore
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| jmp >2
|1:
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
|2:
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
| EXT_CALL zend_compare, r0
if ((opline->opcode != ZEND_CASE &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
| mov dword T1, eax // save
if (opline->opcode != ZEND_CASE) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, NULL
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, NULL
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| mov eax, dword T1 // restore
} else if (may_throw) {
#if ZTS
| mov dword T1, eax // save
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) {
| mov dword T1, eax // save
}
#endif
zend_jit_check_exception_undef_result(Dst, opline);
#if ZTS
| mov eax, dword T1 // restore
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(exception)))) {
| mov eax, dword T1 // restore
}
#endif
}
if (!zend_jit_cmp_slow(Dst, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
if (has_slow) {
| jmp >6
|.code
}
}
|6:
return 1;
}
static int zend_jit_identical(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_ssa_range *op1_range,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_ssa_range *op2_range,
zend_jit_addr op2_addr,
zend_jit_addr res_addr,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr,
bool skip_comparison)
{
uint32_t identical_label = (uint32_t)-1;
uint32_t not_identical_label = (uint32_t)-1;
if (smart_branch_opcode && !exit_addr) {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
if (smart_branch_opcode == ZEND_JMPZ) {
not_identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
identical_label = target_label;
} else {
ZEND_UNREACHABLE();
}
} else {
if (smart_branch_opcode == ZEND_JMPZ) {
identical_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
not_identical_label = target_label;
} else {
ZEND_UNREACHABLE();
}
}
}
if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) {
if (!zend_jit_cmp_long_long(Dst, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison)) {
return 0;
}
return 1;
} else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE &&
(op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) {
if (!zend_jit_cmp_double_double(Dst, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr)) {
return 0;
}
return 1;
}
if ((op1_info & MAY_BE_UNDEF) && (op2_info & MAY_BE_UNDEF)) {
op1_info |= MAY_BE_NULL;
op2_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_Z_TYPE FCARG1a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| IF_Z_TYPE FCARG2a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov aword T1, FCARG1a // save
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| mov FCARG1a, aword T1 // restore
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
} else if (op1_info & MAY_BE_UNDEF) {
op1_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_Z_TYPE FCARG1a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
if (opline->op2_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
} else if (op2_info & MAY_BE_UNDEF) {
op2_info |= MAY_BE_NULL;
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| IF_Z_TYPE FCARG2a, IS_UNDEF, >1
|.cold_code
|1:
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| jmp >1
|.code
|1:
if (opline->op1_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
} else if ((op1_info & op2_info & MAY_BE_ANY) != 0) {
if (opline->op1_type != IS_CONST) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, real_addr, op1_info, 1)) {
return 0;
}
op1_addr = real_addr;
}
}
if (opline->op2_type != IS_CONST) {
if (Z_MODE(op2_addr) == IS_REG) {
zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
if (!zend_jit_spill_store(Dst, op2_addr, real_addr, op2_info, 1)) {
return 0;
}
op2_addr = real_addr;
}
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
if (opline->op1_type != IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
}
if ((op1_info & op2_info & MAY_BE_ANY) == 0) {
if ((opline->opcode != ZEND_CASE_STRICT &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
if (opline->opcode != ZEND_CASE_STRICT) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
}
if (smart_branch_opcode) {
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE)
if (may_throw) {
zend_jit_check_exception(Dst);
}
}
return 1;
}
if (opline->op1_type & (IS_CV|IS_VAR)) {
| ZVAL_DEREF FCARG1a, op1_info
}
if (opline->op2_type & (IS_CV|IS_VAR)) {
| ZVAL_DEREF FCARG2a, op2_info
}
if (has_concrete_type(op1_info)
&& has_concrete_type(op2_info)
&& concrete_type(op1_info) == concrete_type(op2_info)
&& concrete_type(op1_info) <= IS_TRUE) {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE)
}
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) {
if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE)
}
} else {
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, (opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE)
}
}
} else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op1_addr);
| cmp byte [FCARG2a + offsetof(zval, u1.v.type)], Z_TYPE_P(val)
if (smart_branch_opcode) {
if (opline->op2_type == IS_VAR && (op2_info & MAY_BE_REF)) {
| jne >8
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
} else {
| jmp >9
}
|8:
} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| je &exit_addr
} else if (identical_label != (uint32_t)-1) {
| je =>identical_label
} else {
| je >9
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| sete al
} else {
| setne al
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
}
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (smart_branch_opcode && not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
} else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) {
zval *val = Z_ZV(op2_addr);
| cmp byte [FCARG1a + offsetof(zval, u1.v.type)], Z_TYPE_P(val)
if (smart_branch_opcode) {
if (opline->opcode != ZEND_CASE_STRICT
&& opline->op1_type == IS_VAR && (op1_info & MAY_BE_REF)) {
| jne >8
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
} else {
| jmp >9
}
|8:
} else if (exit_addr && smart_branch_opcode == ZEND_JMPNZ) {
| je &exit_addr
} else if (identical_label != (uint32_t)-1) {
| je =>identical_label
} else {
| je >9
}
} else {
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| sete al
} else {
| setne al
}
| movzx eax, al
| lea eax, [eax + 2]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (opline->opcode != ZEND_CASE_STRICT
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
}
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jmp =>not_identical_label
}
}
} else {
if (opline->op1_type == IS_CONST) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_CONST) {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
| EXT_CALL zend_is_identical, r0
if ((opline->opcode != ZEND_CASE_STRICT &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) ||
((opline->op2_type & (IS_VAR|IS_TMP_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)))) {
| mov aword T1, r0 // save
if (opline->opcode != ZEND_CASE_STRICT) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 1, opline
if (may_throw) {
zend_jit_check_exception_undef_result(Dst, opline);
}
| mov r0, aword T1 // restore
}
if (smart_branch_opcode) {
| test al, al
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jnz &exit_addr
} else {
| jz &exit_addr
}
} else if (not_identical_label != (uint32_t)-1) {
| jz =>not_identical_label
if (identical_label != (uint32_t)-1) {
| jmp =>identical_label
}
} else if (identical_label != (uint32_t)-1) {
| jnz =>identical_label
}
} else {
| movzx eax, al
if (opline->opcode != ZEND_IS_NOT_IDENTICAL) {
| lea eax, [eax + 2]
} else {
| neg eax
| lea eax, [eax + 3]
}
| SET_ZVAL_TYPE_INFO res_addr, eax
}
}
|9:
if (may_throw) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_bool_jmpznz(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr)
{
uint32_t true_label = -1;
uint32_t false_label = -1;
bool set_bool = 0;
bool set_bool_not = 0;
bool set_delayed = 0;
bool jmp_done = 0;
if (branch_opcode == ZEND_BOOL) {
set_bool = 1;
} else if (branch_opcode == ZEND_BOOL_NOT) {
set_bool = 1;
set_bool_not = 1;
} else if (branch_opcode == ZEND_JMPZ) {
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ) {
true_label = target_label;
} else if (branch_opcode == ZEND_JMPZ_EX) {
set_bool = 1;
false_label = target_label;
} else if (branch_opcode == ZEND_JMPNZ_EX) {
set_bool = 1;
true_label = target_label;
} else {
ZEND_UNREACHABLE();
}
if (Z_MODE(op1_addr) == IS_CONST_ZVAL) {
if (zend_is_true(Z_ZV(op1_addr))) {
/* Always TRUE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
}
if (true_label != (uint32_t)-1) {
| jmp =>true_label;
}
} else {
/* Always FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
}
if (false_label != (uint32_t)-1) {
| jmp =>false_label;
}
}
return 1;
}
if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) {
if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) {
/* Always TRUE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
}
if (true_label != (uint32_t)-1) {
| jmp =>true_label;
}
} else {
if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) {
/* Always FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
}
} else {
| CMP_ZVAL_TYPE op1_addr, IS_TRUE
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) {
if ((op1_info & MAY_BE_LONG) &&
!(op1_info & MAY_BE_UNDEF) &&
!set_bool) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ) {
| jl >9
} else {
| jl &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| jl =>false_label
} else {
| jl >9
}
jmp_done = 1;
} else {
| jg >2
}
}
if (!(op1_info & MAY_BE_TRUE)) {
/* It's FALSE */
if (set_bool) {
if (set_bool_not) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
}
} else {
if (exit_addr) {
if (set_bool) {
| jne >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jmp &exit_addr
} else {
| jmp >9
}
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| jne &exit_addr
}
}
} else {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| je &exit_addr
} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| jne &exit_addr
} else {
| je >9
}
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (set_bool) {
| jne >1
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
if (true_label != (uint32_t)-1) {
| jmp =>true_label
} else {
| jmp >9
}
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
if (true_label != (uint32_t)-1) {
| je =>true_label
} else if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_LONG))) {
| jne =>false_label
jmp_done = 1;
} else {
| je >9
}
}
} else if (set_bool) {
| sete al
| movzx eax, al
if (set_bool_not) {
| neg eax
| add eax, 3
} else {
| add eax, 2
}
if ((op1_info & MAY_BE_UNDEF) && (op1_info & MAY_BE_ANY)) {
set_delayed = 1;
} else {
| SET_ZVAL_TYPE_INFO res_addr, eax
}
}
}
}
/* It's FALSE, but may be UNDEF */
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & MAY_BE_ANY) {
if (set_delayed) {
| CMP_ZVAL_TYPE op1_addr, IS_UNDEF
| SET_ZVAL_TYPE_INFO res_addr, eax
| jz >1
} else {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
|.cold_code
|1:
}
| mov FCARG1d, opline->op1.var
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_undefined_op_helper, r0
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) {
| jmp &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| jmp =>false_label
}
if (op1_info & MAY_BE_ANY) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jmp >9
}
} else if (false_label == (uint32_t)-1) {
| jmp >9
}
|.code
}
}
if (!jmp_done) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
if (op1_info & MAY_BE_LONG) {
| jmp >9
}
} else if (op1_info & MAY_BE_LONG) {
| jmp &exit_addr
}
} else if (false_label != (uint32_t)-1) {
| jmp =>false_label
} else if ((op1_info & MAY_BE_LONG) || (op1_info & MAY_BE_ANY) == MAY_BE_DOUBLE) {
| jmp >9
}
}
}
}
if (op1_info & MAY_BE_LONG) {
|2:
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >2
}
if (Z_MODE(op1_addr) == IS_REG) {
| test Ra(Z_REG(op1_addr)), Ra(Z_REG(op1_addr))
} else {
| LONG_OP_WITH_CONST, cmp, op1_addr, Z_L(0)
}
if (set_bool) {
| setne al
| movzx eax, al
if (set_bool_not) {
| neg eax
| add eax, 3
} else {
| lea eax, [eax + 2]
}
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jne &exit_addr
} else {
| je &exit_addr
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| jmp =>false_label
}
} else {
| je =>false_label
}
}
}
if ((op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) == MAY_BE_DOUBLE) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.cold_code
}
|2:
if (CAN_USE_AVX()) {
| vxorps xmm0, xmm0, xmm0
} else {
| xorps xmm0, xmm0
}
| DOUBLE_CMP ZREG_XMM0, op1_addr
if (set_bool) {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jp &exit_addr
| jne &exit_addr
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else {
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| je &exit_addr
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
} else if (false_label != (uint32_t)-1) { // JMPZ_EX (p=>true, z=>false, false=>jmp)
| jp >1
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| je => false_label
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
} else if (true_label != (uint32_t)-1) { // JMPNZ_EX (p=>true, z=>false, true=>jmp)
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jp => true_label
| jne => true_label
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
} else if (set_bool_not) { // BOOL_NOT (p=>false, z=>true)
| mov eax, IS_FALSE
| jp >1
| jne >1
| mov eax, IS_TRUE
|1:
| SET_ZVAL_TYPE_INFO res_addr, eax
} else { // BOOL (p=>true, z=>false)
| mov eax, IS_TRUE
| jp >1
| jne >1
| mov eax, IS_FALSE
|1:
| SET_ZVAL_TYPE_INFO res_addr, eax
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
|.code
}
} else {
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jp &exit_addr
| jne &exit_addr
|1:
} else {
| jp >1
| je &exit_addr
|1:
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
ZEND_ASSERT(true_label != (uint32_t)-1 || false_label != (uint32_t)-1);
if (false_label != (uint32_t)-1 ) {
| jp >1
| je => false_label
|1:
if (true_label != (uint32_t)-1) {
| jmp =>true_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| jp => true_label
| jne => true_label
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.code
}
}
} else if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) {
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.cold_code
|2:
}
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_is_true, r0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, >3
}
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
| jnz >3
| mov aword T1, r0 // save
| ZVAL_DTOR_FUNC op1_info, opline
| mov r0, aword T1 // restore
|3:
}
if (may_throw) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r1
| jne ->exception_handler_undef
}
if (set_bool) {
if (set_bool_not) {
| neg eax
| add eax, 3
} else {
| add eax, 2
}
| SET_ZVAL_TYPE_INFO res_addr, eax
if (exit_addr) {
| CMP_ZVAL_TYPE res_addr, IS_FALSE
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jne &exit_addr
} else {
| je &exit_addr
}
} else if (true_label != (uint32_t)-1 || false_label != (uint32_t)-1) {
| CMP_ZVAL_TYPE res_addr, IS_FALSE
if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| jmp =>false_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| je =>false_label
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
|.code
}
} else {
| test r0, r0
if (exit_addr) {
if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) {
| jne &exit_addr
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| je &exit_addr
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
}
} else if (true_label != (uint32_t)-1) {
| jne =>true_label
if (false_label != (uint32_t)-1) {
| jmp =>false_label
} else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
} else {
| je =>false_label
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
| jmp >9
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG)) {
|.code
}
}
}
|9:
return 1;
}
static int zend_jit_qm_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr)
{
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr = op1_def_addr;
}
}
if (!zend_jit_simple_assign(Dst, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 0, 0, 1)) {
return 0;
}
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
if (op1_info & MAY_BE_UNDEF) {
zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_assign(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, int may_throw)
{
ZEND_ASSERT(opline->op1_type == IS_CV);
if (op2_addr != op2_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op2.var, op2_addr, op2_def_addr, op2_info)) {
return 0;
}
if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) {
op2_addr = op2_def_addr;
}
}
if (Z_MODE(op1_addr) != IS_REG
&& Z_MODE(op1_use_addr) == IS_REG
&& !Z_LOAD(op1_use_addr)
&& !Z_STORE(op1_use_addr)) {
/* Force type update */
op1_info |= MAY_BE_UNDEF;
}
if (!zend_jit_assign_to_variable(Dst, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr,
may_throw)) {
return 0;
}
if (!zend_jit_store_var_if_necessary_ex(Dst, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) {
return 0;
}
if (opline->result_type != IS_UNUSED) {
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
}
return 1;
}
/* copy of hidden zend_closure */
typedef struct _zend_closure {
zend_object std;
zend_function func;
zval this_ptr;
zend_class_entry *called_scope;
zif_handler orig_internal_handler;
} zend_closure;
static int zend_jit_stack_check(dasm_State **Dst, const zend_op *opline, uint32_t used_stack)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| // Check Stack Overflow
| MEM_LOAD_ZTS r1, aword, executor_globals, vm_stack_end, r0
| MEM_LOAD_OP_ZTS sub, r1, aword, executor_globals, vm_stack_top, r0
| cmp r1, used_stack
| jb &exit_addr
return 1;
}
static int zend_jit_push_call_frame(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack)
{
uint32_t used_stack;
bool stack_check = 1;
const size_t func_type_offset = is_closure ? offsetof(zend_closure, func.type) : offsetof(zend_function, type);
if (func) {
used_stack = zend_vm_calc_used_stack(opline->extended_value, func);
if ((int)used_stack <= checked_stack) {
stack_check = 0;
}
} else {
used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval);
| // if (EXPECTED(ZEND_USER_CODE(func->type))) {
| test byte [r0 + func_type_offset], 1
| mov FCARG1a, used_stack
| jnz >1
| // used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval);
| mov edx, opline->extended_value
if (!is_closure) {
| cmp edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
| cmova edx, dword [r0 + offsetof(zend_function, op_array.num_args)]
| sub edx, dword [r0 + offsetof(zend_function, op_array.last_var)]
| sub edx, dword [r0 + offsetof(zend_function, op_array.T)]
} else {
| cmp edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)]
| cmova edx, dword [r0 + offsetof(zend_closure, func.op_array.num_args)]
| sub edx, dword [r0 + offsetof(zend_closure, func.op_array.last_var)]
| sub edx, dword [r0 + offsetof(zend_closure, func.op_array.T)]
}
| shl edx, 4
|.if X64
| movsxd r2, edx
|.endif
| sub FCARG1a, r2
|1:
}
zend_jit_start_reuse_ip();
| // if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) {
| MEM_LOAD_ZTS RX, aword, executor_globals, vm_stack_top, RX
if (stack_check) {
| // Check Stack Overflow
| MEM_LOAD_ZTS r2, aword, executor_globals, vm_stack_end, r2
| sub r2, RX
if (func) {
| cmp r2, used_stack
} else {
| cmp r2, FCARG1a
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jb &exit_addr
} else {
| jb >1
| // EG(vm_stack_top) = (zval*)((char*)call + used_stack);
|.cold_code
|1:
if (func) {
| mov FCARG1d, used_stack
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_int_extend_stack_helper, r0
} else {
if (!is_closure) {
if (func
&& op_array == &func->op_array
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
&& (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) {
| LOAD_ADDR FCARG2a, func
} else {
| mov FCARG2a, r0
}
} else {
| lea FCARG2a, aword [r0 + offsetof(zend_closure, func)]
}
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_extend_stack_helper, r0
}
| mov RX, r0
| jmp >1
|.code
}
}
if (func) {
| MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, used_stack, r2
} else {
| MEM_UPDATE_ZTS add, aword, executor_globals, vm_stack_top, FCARG1a, r2
}
| // zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object);
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) {
| // ZEND_SET_CALL_INFO(call, 0, call_info);
| mov dword EX:RX->This.u1.type_info, (IS_UNDEF | ZEND_CALL_NESTED_FUNCTION)
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) {
#endif
| // call->func = func;
|1:
| ADDR_STORE aword EX:RX->func, func, r1
} else {
if (!is_closure) {
| // call->func = func;
if (func
&& op_array == &func->op_array
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)
&& (sizeof(void*) != 8 || IS_SIGNED_32BIT(func))) {
| ADDR_STORE aword EX:RX->func, func, r1
} else {
| mov aword EX:RX->func, r0
}
} else {
| // call->func = &closure->func;
| lea r1, aword [r0 + offsetof(zend_closure, func)]
| mov aword EX:RX->func, r1
}
|1:
}
if (opline->opcode == ZEND_INIT_METHOD_CALL) {
| // Z_PTR(call->This) = obj;
| mov r1, aword T1
| mov aword EX:RX->This.value.ptr, r1
if (opline->op1_type == IS_UNUSED || delayed_fetch_this) {
| // call->call_info |= ZEND_CALL_HAS_THIS;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| mov dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS
} else {
| or dword EX:RX->This.u1.type_info, ZEND_CALL_HAS_THIS
}
} else {
if (opline->op1_type == IS_CV) {
| // GC_ADDREF(obj);
| add dword [r1], 1
}
| // call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| mov dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)
} else {
| or dword EX:RX->This.u1.type_info, (ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)
}
}
} else if (!is_closure) {
| // Z_CE(call->This) = called_scope;
| mov aword EX:RX->This.value.ptr, 0
} else {
if (opline->op2_type == IS_CV) {
| // GC_ADDREF(closure);
| add dword [r0], 1
}
| // object_or_called_scope = closure->called_scope;
| mov r1, aword [r0 + offsetof(zend_closure, called_scope)]
| mov aword EX:RX->This.value.ptr, r1
| // call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE |
| // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE);
| mov edx, dword [r0 + offsetof(zend_closure, func.common.fn_flags)]
| and edx, ZEND_ACC_FAKE_CLOSURE
| or edx, (ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE)
| // if (Z_TYPE(closure->this_ptr) != IS_UNDEF) {
| cmp byte [r0 + offsetof(zend_closure, this_ptr.u1.v.type)], IS_UNDEF
| jz >1
| // call_info |= ZEND_CALL_HAS_THIS;
| or edx, ZEND_CALL_HAS_THIS
| // object_or_called_scope = Z_OBJ(closure->this_ptr);
| mov r1, aword [r0 + offsetof(zend_closure, this_ptr.value.ptr)]
|1:
| // ZEND_SET_CALL_INFO(call, 0, call_info);
| or dword EX:RX->This.u1.type_info, edx
| // Z_PTR(call->This) = object_or_called_scope;
| mov aword EX:RX->This.value.ptr, r1
if (!func) {
| cmp byte [r0 + offsetof(zend_closure, func.type)], ZEND_USER_FUNCTION
| jnz >1
}
if (!func || func->common.type == ZEND_USER_FUNCTION) {
| lea FCARG1a, aword [r0 + offsetof(zend_closure, func)]
| EXT_CALL zend_jit_init_func_run_time_cache_helper, r0
}
if (!func) {
|1:
}
}
| // ZEND_CALL_NUM_ARGS(call) = num_args;
| mov dword EX:RX->This.u2.num_args, opline->extended_value
return 1;
}
static int zend_jit_init_fcall_guard(dasm_State **Dst, uint32_t level, const zend_function *func, const zend_op *to_opline)
{
int32_t exit_point;
const void *exit_addr;
if (func->type == ZEND_USER_FUNCTION) {
if (!zend_accel_in_shm(func->op_array.opcodes)) {
/* op_array and op_array->opcodes are not persistent. We can't link. */
return 0;
}
}
exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| // call = EX(call);
| mov r1, EX->call
while (level > 0) {
| mov r1, EX:r1->prev_execute_data
level--;
}
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
| mov r1, aword EX:r1->func
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 r2, ((ptrdiff_t)opcodes)
| cmp aword [r1 + offsetof(zend_op_array, opcodes)], r2
|| } else {
| cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes
|| }
| .else
| cmp aword [r1 + offsetof(zend_op_array, opcodes)], opcodes
| .endif
| jne &exit_addr
#ifdef _WIN32
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
const zif_handler handler = func->internal_function.handler;
| .if X64
|| if (!IS_SIGNED_32BIT(handler)) {
| mov64 r2, ((ptrdiff_t)handler)
| cmp aword [r1 + offsetof(zend_internal_function, handler)], r2
|| } else {
| cmp aword [r1 + offsetof(zend_internal_function, handler)], handler
|| }
| .else
| cmp aword [r1 + offsetof(zend_internal_function, handler)], handler
| .endif
| jne &exit_addr
#endif
} else {
| .if X64
|| if (!IS_SIGNED_32BIT(func)) {
| mov64 r2, ((ptrdiff_t)func)
| cmp aword EX:r1->func, r2
|| } else {
| cmp aword EX:r1->func, func
|| }
| .else
| cmp aword EX:r1->func, func
| .endif
| jne &exit_addr
}
return 1;
}
static int zend_jit_init_fcall(dasm_State **Dst, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (!func
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL) {
#ifdef _WIN32
/* ASLR */
if (trace->func->type != ZEND_INTERNAL_FUNCTION) {
func = (zend_function*)trace->func;
}
#else
func = (zend_function*)trace->func;
#endif
}
#ifdef _WIN32
if (0) {
#else
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_INTERNAL_FUNCTION) {
#endif
/* load constant address later */
} else if (func && op_array == &func->op_array) {
/* recursive call */
if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) ||
(sizeof(void*) == 8 && !IS_SIGNED_32BIT(func))) {
| mov r0, EX->func
}
} else {
| // if (CACHED_PTR(opline->result.num))
| mov r2, EX->run_time_cache
| mov r0, aword [r2 + opline->result.num]
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& func
&& (func->common.fn_flags & ZEND_ACC_IMMUTABLE)
&& opline->opcode != ZEND_INIT_FCALL) {
/* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */
| .if X64
|| if (!IS_SIGNED_32BIT(func)) {
| mov64 r1, ((ptrdiff_t)func)
| cmp r0, r1
|| } else {
| cmp r0, func
|| }
| .else
| cmp r0, func
| .endif
| jnz >1
|.cold_code
|1:
} else {
| test r0, r0
| jz >1
}
|.cold_code
|1:
if (opline->opcode == ZEND_INIT_FCALL
&& func
&& func->type == ZEND_USER_FUNCTION
&& (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) {
| LOAD_ADDR FCARG1a, func
| mov aword [r2 + opline->result.num], FCARG1a
| EXT_CALL zend_jit_init_func_run_time_cache_helper, r0
| jmp >3
} else {
zval *zv = RT_CONSTANT(opline, opline->op2);
if (opline->opcode == ZEND_INIT_FCALL) {
| LOAD_ADDR FCARG1a, Z_STR_P(zv);
| lea FCARG2a, aword [r2 + opline->result.num]
| EXT_CALL zend_jit_find_func_helper, r0
} else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) {
| LOAD_ADDR FCARG1a, Z_STR_P(zv + 1);
| lea FCARG2a, aword [r2 + opline->result.num]
| EXT_CALL zend_jit_find_func_helper, r0
} else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) {
| LOAD_ADDR FCARG1a, zv;
| lea FCARG2a, aword [r2 + opline->result.num]
| EXT_CALL zend_jit_find_ns_func_helper, r0
} else {
ZEND_UNREACHABLE();
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline,
func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (!func || opline->opcode == ZEND_INIT_FCALL) {
| test r0, r0
| jnz >3
} else if (func->type == ZEND_USER_FUNCTION
&& !(func->common.fn_flags & ZEND_ACC_IMMUTABLE)) {
const zend_op *opcodes = func->op_array.opcodes;
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 r1, ((ptrdiff_t)opcodes)
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1
|| } else {
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
|| }
| .else
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
| .endif
| jz >3
#ifdef _WIN32
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
const zif_handler handler = func->internal_function.handler;
| .if X64
|| if (!IS_SIGNED_32BIT(handler)) {
| mov64 r1, ((ptrdiff_t)handler)
| cmp aword [r0 + offsetof(zend_internal_function, handler)], r1
|| } else {
| cmp aword [r0 + offsetof(zend_internal_function, handler)], handler
|| }
| .else
| cmp aword [r0 + offsetof(zend_internal_function, handler)], handler
| .endif
| jz >3
#endif
} else {
| .if X64
|| if (!IS_SIGNED_32BIT(func)) {
| mov64 r1, ((ptrdiff_t)func)
| cmp r0, r1
|| } else {
| cmp r0, func
|| }
| .else
| cmp r0, func
| .endif
| jz >3
}
| jmp &exit_addr
} else {
| test r0, r0
| jnz >3
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, r0
| jmp ->undefined_function
}
}
|.code
|3:
}
if (!zend_jit_push_call_frame(Dst, opline, op_array, func, 0, 0, checked_stack)) {
return 0;
}
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
return 1;
}
static int zend_jit_init_method_call(dasm_State **Dst,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
uint32_t op1_info,
zend_jit_addr op1_addr,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
zend_jit_trace_rec *trace,
int checked_stack,
bool polymorphic_side_trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
zend_function *func = NULL;
zval *function_name;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
function_name = RT_CONSTANT(opline, opline->op2);
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_init_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
}
if (polymorphic_side_trace) {
/* function is passed in r0 from parent_trace */
} else {
if (on_this) {
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
} else {
/* Hack: Convert reference to regular value to simplify JIT code */
ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP);
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| EXT_CALL zend_jit_unref_helper, r0
|1:
}
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| SET_EX_OPLINE opline, r0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
| EXT_CALL zend_jit_invalid_method_call_tmp, r0
} else {
| EXT_CALL zend_jit_invalid_method_call, r0
}
| jmp ->exception_handler
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
| mov aword T1, FCARG1a // save
if (func) {
| // fbc = CACHED_PTR(opline->result.num + sizeof(void*));
| mov r0, EX->run_time_cache
| mov r0, aword [r0 + opline->result.num + sizeof(void*)]
| test r0, r0
| jz >1
} else {
| // if (CACHED_PTR(opline->result.num) == obj->ce)) {
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + opline->result.num]
| cmp r2, [FCARG1a + offsetof(zend_object, ce)]
| jnz >1
| // fbc = CACHED_PTR(opline->result.num + sizeof(void*));
| mov r0, aword [r0 + opline->result.num + sizeof(void*)]
}
|.cold_code
|1:
| LOAD_ADDR FCARG2a, function_name
|.if X64
| lea CARG3, aword T1
|.else
| lea r0, aword T1
| sub r4, 12
| push r0
|.endif
| SET_EX_OPLINE opline, r0
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
| EXT_CALL zend_jit_find_method_tmp_helper, r0
} else {
| EXT_CALL zend_jit_find_method_helper, r0
}
|.if not(X64)
| add r4, 12
|.endif
| test r0, r0
| jnz >2
| jmp ->exception_handler
|.code
|2:
}
if ((!func || zend_jit_may_be_modified(func, op_array))
&& trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func) {
int32_t exit_point;
const void *exit_addr;
exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
func = (zend_function*)trace->func;
if (func->type == ZEND_USER_FUNCTION &&
(!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) ||
(func->common.fn_flags & ZEND_ACC_CLOSURE) ||
!func->common.function_name)) {
const zend_op *opcodes = func->op_array.opcodes;
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 r1, ((ptrdiff_t)opcodes)
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], r1
|| } else {
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
|| }
| .else
| cmp aword [r0 + offsetof(zend_op_array, opcodes)], opcodes
| .endif
| jne &exit_addr
#ifdef _WIN32
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
const zif_handler handler = func->internal_function.handler;
| .if X64
|| if (!IS_SIGNED_32BIT(handler)) {
| mov64 r1, ((ptrdiff_t)handler)
| cmp aword [r0 + offsetof(zend_internal_function, handler)], r1
|| } else {
| cmp aword [r0 + offsetof(zend_internal_function, handler)], handler
|| }
| .else
| cmp aword [r0 + offsetof(zend_internal_function, handler)], handler
| .endif
| jne &exit_addr
#endif
} else {
| .if X64
|| if (!IS_SIGNED_32BIT(func)) {
| mov64 r1, ((ptrdiff_t)func)
| cmp r0, r1
|| } else {
| cmp r0, func
|| }
| .else
| cmp r0, func
| .endif
| jne &exit_addr
}
}
if (!func) {
| // if (fbc->common.fn_flags & ZEND_ACC_STATIC) {
| test dword [r0 + offsetof(zend_function, common.fn_flags)], ZEND_ACC_STATIC
| jnz >1
|.cold_code
|1:
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) {
| mov FCARG1a, aword T1 // restore
| mov FCARG2a, r0
|.if X64
| mov CARG3d, opline->extended_value
|.else
| sub r4, 12
| push opline->extended_value
|.endif
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) {
| EXT_CALL zend_jit_push_static_metod_call_frame_tmp, r0
} else {
| EXT_CALL zend_jit_push_static_metod_call_frame, r0
}
|.if not(X64)
| add r4, 12
|.endif
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) {
| test r0, r0
| jz ->exception_handler
}
| mov RX, r0
}
if (!func) {
| jmp >9
|.code
}
if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) {
if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 0, delayed_fetch_this, checked_stack)) {
return 0;
}
}
if (!func) {
|9:
}
zend_jit_start_reuse_ip();
if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
return 1;
}
static int zend_jit_init_closure_call(dasm_State **Dst,
const zend_op *opline,
uint32_t b,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
int call_level,
zend_jit_trace_rec *trace,
int checked_stack)
{
zend_function *func = NULL;
zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
| GET_ZVAL_PTR r0, op2_addr
if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure
&& !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
|.if X64
|| if (!IS_SIGNED_32BIT(zend_ce_closure)) {
| mov64 FCARG1a, ((ptrdiff_t)zend_ce_closure)
| cmp aword [r0 + offsetof(zend_object, ce)], FCARG1a
|| } else {
| cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure
|| }
|.else
| cmp aword [r0 + offsetof(zend_object, ce)], zend_ce_closure
|.endif
| jne &exit_addr
if (ssa->var_info && ssa_op->op2_use >= 0) {
ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure;
ssa->var_info[ssa_op->op2_use].is_instanceof = 0;
}
}
if (trace
&& trace->op == ZEND_JIT_TRACE_INIT_CALL
&& trace->func
&& trace->func->type == ZEND_USER_FUNCTION) {
const zend_op *opcodes;
int32_t exit_point;
const void *exit_addr;
func = (zend_function*)trace->func;
opcodes = func->op_array.opcodes;
exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| .if X64
|| if (!IS_SIGNED_32BIT(opcodes)) {
| mov64 FCARG1a, ((ptrdiff_t)opcodes)
| cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], FCARG1a
|| } else {
| cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes
|| }
| .else
| cmp aword [r0 + offsetof(zend_closure, func.op_array.opcodes)], opcodes
| .endif
| jne &exit_addr
}
if (delayed_call_chain) {
if (!zend_jit_save_call_chain(Dst, delayed_call_level)) {
return 0;
}
}
if (!zend_jit_push_call_frame(Dst, opline, NULL, func, 1, 0, checked_stack)) {
return 0;
}
if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) {
if (!zend_jit_save_call_chain(Dst, call_level)) {
return 0;
}
} else {
delayed_call_chain = 1;
delayed_call_level = call_level;
}
if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
if (!zend_jit_set_valid_ip(Dst, opline + 1)) {
return 0;
}
}
return 1;
}
static int zend_jit_do_fcall(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace)
{
zend_func_info *info = ZEND_FUNC_INFO(op_array);
zend_call_info *call_info = NULL;
const zend_function *func = NULL;
uint32_t i;
zend_jit_addr res_addr;
uint32_t call_num_args = 0;
bool unknown_num_args = 0;
const void *exit_addr = NULL;
const zend_op *prev_opline;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
} else {
/* CPU stack allocated temporary zval */
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R4, TMP_ZVAL_OFFSET);
}
prev_opline = opline - 1;
while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) {
prev_opline--;
}
if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
unknown_num_args = 1;
}
if (info) {
call_info = info->callee_info;
while (call_info && call_info->caller_call_opline != opline) {
call_info = call_info->next_callee;
}
if (call_info && call_info->callee_func && !call_info->is_prototype) {
func = call_info->callee_func;
}
if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE)
&& (!JIT_G(current_frame) ||
!JIT_G(current_frame)->call ||
!JIT_G(current_frame)->call->func)) {
call_info = NULL; func = NULL; /* megamorphic call from trait */
}
}
if (!func) {
/* resolve function at run time */
} else if (func->type == ZEND_USER_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL);
call_num_args = call_info->num_args;
} else if (func->type == ZEND_INTERNAL_FUNCTION) {
ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL);
call_num_args = call_info->num_args;
} else {
ZEND_UNREACHABLE();
}
if (trace && !func) {
if (trace->op == ZEND_JIT_TRACE_DO_ICALL) {
ZEND_ASSERT(trace->func->type == ZEND_INTERNAL_FUNCTION);
#ifndef ZEND_WIN32
// TODO: ASLR may cause different addresses in different workers ???
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
#endif
} else if (trace->op == ZEND_JIT_TRACE_ENTER) {
ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION);
if (zend_accel_in_shm(trace->func->op_array.opcodes)) {
func = trace->func;
if (JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) {
call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call);
} else {
unknown_num_args = 1;
}
}
}
}
bool may_have_extra_named_params =
opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS &&
(!func || func->common.fn_flags & ZEND_ACC_VARIADIC);
if (!reuse_ip) {
zend_jit_start_reuse_ip();
| // call = EX(call);
| mov RX, EX->call
}
zend_jit_stop_reuse_ip();
| // fbc = call->func;
| // mov r2, EX:RX->func ???
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, r0
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz &exit_addr
}
}
}
if (!delayed_call_chain) {
if (call_level == 1) {
| mov aword EX->call, 0
} else {
| //EX(call) = call->prev_execute_data;
| mov r0, EX:RX->prev_execute_data
| mov EX->call, r0
}
}
delayed_call_chain = 0;
| //call->prev_execute_data = execute_data;
| mov EX:RX->prev_execute_data, EX
if (!func) {
| mov r0, EX:RX->func
}
if (opline->opcode == ZEND_DO_FCALL) {
if (!func) {
if (!trace) {
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| mov r0, EX:RX->func // reload
| jne >1
| jmp ->exception_handler
|.code
|1:
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| je ->exception_handler
}
}
if (!func
&& opline->opcode != ZEND_DO_UCALL
&& opline->opcode != ZEND_DO_ICALL) {
| cmp byte [r0 + offsetof(zend_function, type)], ZEND_USER_FUNCTION
| jne >8
}
if ((!func || func->type == ZEND_USER_FUNCTION)
&& opline->opcode != ZEND_DO_ICALL) {
| // EX(call) = NULL;
| mov aword EX:RX->call, 0
if (RETURN_VALUE_USED(opline)) {
| // EX(return_value) = EX_VAR(opline->result.var);
| LOAD_ZVAL_ADDR r2, res_addr
| mov aword EX:RX->return_value, r2
} else {
| // EX(return_value) = 0;
| mov aword EX:RX->return_value, 0
}
//EX_LOAD_RUN_TIME_CACHE(op_array);
if (!func || func->op_array.cache_size) {
if (func && op_array == &func->op_array) {
/* recursive call */
if (trace || func->op_array.cache_size > sizeof(void*)) {
| mov r2, EX->run_time_cache
| mov EX:RX->run_time_cache, r2
}
} else {
if (func
&& !(func->op_array.fn_flags & ZEND_ACC_CLOSURE)
&& ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) {
| MEM_LOAD_ZTS r2, aword, compiler_globals, map_ptr_base, r1
| mov r2, aword [r2 + (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache)]
} else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) ||
(JIT_G(current_frame) &&
JIT_G(current_frame)->call &&
TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) {
/* Closures always use direct pointers */
| mov r0, EX:RX->func
| mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)]
} else {
if (func) {
| mov r0, EX:RX->func
}
| mov r2, aword [r0 + offsetof(zend_op_array, run_time_cache__ptr)]
| test r2, 1
| jz >1
| MEM_LOAD_OP_ZTS add, r2, aword, compiler_globals, map_ptr_base, r1
| mov r2, aword [r2]
|1:
}
| mov EX:RX->run_time_cache, r2
}
}
| // EG(current_execute_data) = execute_data;
| MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1
| mov FP, RX
| // opline = op_array->opcodes;
if (func && !unknown_num_args) {
for (i = call_num_args; i < func->op_array.last_var; i++) {
uint32_t n = EX_NUM_TO_VAR(i);
| SET_Z_TYPE_INFO RX + n, IS_UNDEF
}
if (call_num_args <= func->op_array.num_args) {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
uint32_t num_args;
if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) {
if (trace) {
num_args = 0;
} else if (call_info) {
num_args = skip_valid_arguments(op_array, ssa, call_info);
} else {
num_args = call_num_args;
}
} else {
num_args = call_num_args;
}
if (zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes + num_args)
} else {
| mov r0, EX->func
if (GCC_GLOBAL_REGS) {
| mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
if (num_args) {
| add IP, (num_args * sizeof(zend_op))
}
} else {
| mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
if (num_args) {
| add FCARG1a, (num_args * sizeof(zend_op))
}
| mov aword EX->opline, FCARG1a
}
}
if (GCC_GLOBAL_REGS && !trace && op_array == &func->op_array
&& num_args >= op_array->required_num_args) {
/* recursive call */
if (ZEND_OBSERVER_ENABLED) {
| SAVE_IP
| mov FCARG1a, FP
| EXT_CALL zend_observer_fcall_begin, r0
}
#ifdef CONTEXT_THREADED_JIT
| call >1
|.cold_code
|1:
| pop r0
| jmp =>num_args
|.code
#else
| jmp =>num_args
#endif
return 1;
}
}
} else {
if (!trace || (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER)) {
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes)
} else if (GCC_GLOBAL_REGS) {
| mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
} else {
| mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
| mov aword EX->opline, FCARG1a
}
}
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL zend_jit_copy_extra_args_helper, r0
}
} else {
| // opline = op_array->opcodes
if (func && zend_accel_in_shm(func->op_array.opcodes)) {
| LOAD_IP_ADDR (func->op_array.opcodes)
} else if (GCC_GLOBAL_REGS) {
| mov IP, aword [r0 + offsetof(zend_op_array, opcodes)]
} else {
| mov FCARG1a, aword [r0 + offsetof(zend_op_array, opcodes)]
| mov aword EX->opline, FCARG1a
}
if (func) {
| // num_args = EX_NUM_ARGS();
| mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
| // if (UNEXPECTED(num_args > first_extra_arg))
| cmp ecx, (func->op_array.num_args)
} else {
| // first_extra_arg = op_array->num_args;
| mov edx, dword [r0 + offsetof(zend_op_array, num_args)]
| // num_args = EX_NUM_ARGS();
| mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)]
| // if (UNEXPECTED(num_args > first_extra_arg))
| cmp ecx, edx
}
| jg >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL zend_jit_copy_extra_args_helper, r0
if (!func) {
| mov r0, EX->func // reload
}
| mov ecx, dword [FP + offsetof(zend_execute_data, This.u2.num_args)] // reload
| jmp >1
|.code
if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) {
if (!func) {
| // if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0))
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_HAS_TYPE_HINTS
| jnz >1
}
| // opline += num_args;
|.if X64
|| ZEND_ASSERT(sizeof(zend_op) == 32);
| mov edx, ecx
| shl r2, 5
|.else
| imul r2, ecx, sizeof(zend_op)
|.endif
| ADD_IP r2
}
|1:
| // if (EXPECTED((int)num_args < op_array->last_var)) {
if (func) {
| mov edx, (func->op_array.last_var)
} else {
| mov edx, dword [r0 + offsetof(zend_op_array, last_var)]
}
| sub edx, ecx
| jle >3 //???
| // zval *var = EX_VAR_NUM(num_args);
// |.if X64
// | movsxd r1, ecx
// |.endif
| shl r1, 4
| lea r1, [FP + r1 + (ZEND_CALL_FRAME_SLOT * sizeof(zval))]
|2:
| SET_Z_TYPE_INFO r1, IS_UNDEF
| sub edx, 1
| lea r1, [r1 + 16]
| jne <2
|3:
}
if (ZEND_OBSERVER_ENABLED) {
if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_INTERPRETER)) {
ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END);
| SET_EX_OPLINE trace[1].opline, r0
} else {
| SAVE_IP
}
| mov FCARG1a, FP
| EXT_CALL zend_observer_fcall_begin, r0
}
if (trace) {
if (!func && (opline->opcode != ZEND_DO_UCALL)) {
| jmp >9
}
} else {
#ifdef CONTEXT_THREADED_JIT
| call ->context_threaded_call
if (!func && (opline->opcode != ZEND_DO_UCALL)) {
| jmp >9
}
| call ->context_threaded_call
if (!func) {
| jmp >9
}
#else
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
| JMP_IP
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
| JMP_IP
} else {
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 1 // ZEND_VM_ENTER
| ret
}
}
#endif
}
if ((!func || func->type == ZEND_INTERNAL_FUNCTION)
&& (opline->opcode != ZEND_DO_UCALL)) {
if (!func && (opline->opcode != ZEND_DO_ICALL)) {
|8:
}
if (opline->opcode == ZEND_DO_FCALL_BY_NAME) {
if (!func) {
if (trace) {
uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz &exit_addr
} else {
| test dword [r0 + offsetof(zend_op_array, fn_flags)], ZEND_ACC_DEPRECATED
| jnz >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| mov r0, EX:RX->func // reload
| jne >1
| jmp ->exception_handler
|.code
|1:
}
} else if (func->common.fn_flags & ZEND_ACC_DEPRECATED) {
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, RX
}
| EXT_CALL zend_jit_deprecated_helper, r0
| test al, al
| je ->exception_handler
| mov r0, EX:RX->func // reload
}
}
| // EG(current_execute_data) = execute_data;
| MEM_STORE_ZTS aword, executor_globals, current_execute_data, RX, r1
if (ZEND_OBSERVER_ENABLED) {
| mov FCARG1a, RX
| EXT_CALL zend_observer_fcall_begin, r0
| mov r0, EX:RX->func // reload
}
| // ZVAL_NULL(EX_VAR(opline->result.var));
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| SET_Z_TYPE_INFO FCARG2a, IS_NULL
zend_jit_reset_last_valid_opline();
| // (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret);
if (zend_execute_internal) {
|.if X64
| // CARG2 and FCARG2a are identical
| mov CARG1, RX
|.else
| mov aword A2, FCARG2a
| mov aword A1, RX
|.endif
| EXT_CALL zend_execute_internal, r0
} else {
| mov FCARG1a, RX
if (func) {
| EXT_CALL func->internal_function.handler, r0
} else {
| call aword [r0 + offsetof(zend_internal_function, handler)]
}
}
if (ZEND_OBSERVER_ENABLED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| mov FCARG1a, RX
| EXT_CALL zend_observer_fcall_end, r0
}
| // EG(current_execute_data) = execute_data;
| MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0
| // zend_vm_stack_free_args(call);
if (func && !unknown_num_args) {
for (i = 0; i < call_num_args; i++ ) {
if (zend_jit_needs_arg_dtor(func, i, call_info)) {
uint32_t offset = EX_NUM_TO_VAR(i);
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, 1, opline
}
}
} else {
| mov FCARG1a, RX
| EXT_CALL zend_jit_vm_stack_free_args_helper, r0
}
if (may_have_extra_named_params) {
| test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24)
| jnz >1
|.cold_code
|1:
| mov FCARG1a, aword [RX + offsetof(zend_execute_data, extra_named_params)]
| EXT_CALL zend_free_extra_named_params, r0
| jmp >2
|.code
|2:
}
|8:
if (opline->opcode == ZEND_DO_FCALL) {
// TODO: optimize ???
| // if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS))
| test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_RELEASE_THIS >> 16)
| jnz >1
|.cold_code
|1:
| GET_Z_PTR FCARG1a, RX + offsetof(zend_execute_data, This)
| // OBJ_RELEASE(object);
| OBJ_RELEASE ZREG_FCARG1, >2
| jmp >2
|.code
|2:
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!JIT_G(current_frame)->call ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) ||
prev_opline->opcode == ZEND_SEND_UNPACK ||
prev_opline->opcode == ZEND_SEND_ARRAY ||
prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) {
| // zend_vm_stack_free_call_frame(call);
| test byte [RX + offsetof(zend_execute_data, This.u1.type_info) + 2], (ZEND_CALL_ALLOCATED >> 16)
| jnz >1
|.cold_code
|1:
| mov FCARG1a, RX
| EXT_CALL zend_jit_free_call_frame, r0
| jmp >1
|.code
}
| MEM_STORE_ZTS aword, executor_globals, vm_stack_top, RX, r0
|1:
if (!RETURN_VALUE_USED(opline)) {
zend_class_entry *ce;
bool ce_is_instanceof;
uint32_t func_info = call_info ?
zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) :
(MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN);
/* If an exception is thrown, the return_value may stay at the
* original value of null. */
func_info |= MAY_BE_NULL;
if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
| ZVAL_PTR_DTOR res_addr, func_info, 1, 1, opline
}
}
| // if (UNEXPECTED(EG(exception) != NULL)) {
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| jne ->icall_throw_handler
// TODO: Can we avoid checking for interrupts after each call ???
if (trace && last_valid_opline != opline) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
exit_addr = NULL;
}
if (!zend_jit_check_timeout(Dst, opline + 1, exit_addr)) {
return 0;
}
if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) {
| LOAD_IP_ADDR (opline + 1)
} else if (trace
&& trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_INTERPRETER) {
| LOAD_IP_ADDR (opline + 1)
}
}
if (!func) {
|9:
}
return 1;
}
static int zend_jit_send_val(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAL_EX) {
uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2);
ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
/* Don't generate code that always throws exception */
return 0;
}
} else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz &exit_addr
} else {
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
if (Z_MODE(op1_addr) == IS_REG) {
/* set type to avoid zval_ptr_dtor() on uninitialized value */
zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
| SET_ZVAL_TYPE_INFO addr, IS_UNDEF
}
| SET_EX_OPLINE opline, r0
| jmp ->throw_cannot_pass_by_ref
|.code
}
}
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
}
return 1;
}
static int zend_jit_check_undef_args(dasm_State **Dst, const zend_op *opline)
{
| mov FCARG1a, EX->call
| test byte [FCARG1a + offsetof(zend_execute_data, This.u1.type_info) + 3], (ZEND_CALL_MAY_HAVE_UNDEF >> 24)
| jnz >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_handle_undef_args, r0
| test r0, r0
| jnz ->exception_handler
| jmp >2
|.code
|2:
return 1;
}
static int zend_jit_send_ref(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold)
{
zend_jit_addr op1_addr, arg_addr, ref_addr;
op1_addr = OP1_ADDR();
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->op1_type == IS_VAR) {
if (op1_info & MAY_BE_INDIRECT) {
| LOAD_ZVAL_ADDR r0, op1_addr
| // if (EXPECTED(Z_TYPE_P(ret) == IS_INDIRECT)) {
| IF_NOT_Z_TYPE r0, IS_INDIRECT, >1
| // ret = Z_INDIRECT_P(ret);
| GET_Z_PTR r0, r0
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
}
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
| jmp >2
|1:
}
op1_info &= ~MAY_BE_UNDEF;
op1_info |= MAY_BE_NULL;
}
} else {
ZEND_UNREACHABLE();
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >2
| GET_ZVAL_PTR r1, op1_addr
| GC_ADDREF r1
| SET_ZVAL_PTR arg_addr, r1
| SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX
| jmp >6
}
|2:
| // ZVAL_NEW_REF(arg, varptr);
if (opline->op1_type == IS_VAR) {
if (Z_REG(op1_addr) != ZREG_R0 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR r0, op1_addr
}
| mov aword T1, r0 // save
}
| EMALLOC sizeof(zend_reference), op_array, opline
| mov dword [r0], 2
| mov dword [r0 + offsetof(zend_reference, gc.u.type_info)], GC_REFERENCE
| mov aword [r0 + offsetof(zend_reference, sources.ptr)], 0
ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val));
if (opline->op1_type == IS_VAR) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
| mov r1, aword T1 // restore
| ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R2, ZREG_R2
| SET_ZVAL_PTR val_addr, r0
| SET_ZVAL_TYPE_INFO val_addr, IS_REFERENCE_EX
} else {
| ZVAL_COPY_VALUE ref_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
| SET_ZVAL_PTR op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
}
| SET_ZVAL_PTR arg_addr, r0
| SET_ZVAL_TYPE_INFO arg_addr, IS_REFERENCE_EX
}
|6:
| FREE_OP opline->op1_type, opline->op1, op1_info, !cold, opline
|7:
return 1;
}
static int zend_jit_send_var(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr)
{
uint32_t arg_num = opline->op2.num;
zend_jit_addr arg_addr;
ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX &&
opline->opcode != ZEND_SEND_VAR_NO_REF_EX) ||
arg_num <= MAX_ARG_FLAG_NUM);
arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
if (opline->opcode == ZEND_SEND_VAR_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
return 0;
}
| jmp >7
|.code
}
} else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!(op1_info & MAY_BE_REF)) {
/* Don't generate code that always throws exception */
return 0;
} else {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| cmp cl, IS_REFERENCE
| jne &exit_addr
}
}
return 1;
}
} else {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2);
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (op1_info & MAY_BE_REF) {
| cmp cl, IS_REFERENCE
| je >7
}
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >7
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jmp &exit_addr
} else {
| SET_EX_OPLINE opline, r0
| LOAD_ZVAL_ADDR FCARG1a, arg_addr
| EXT_CALL zend_jit_only_vars_by_reference, r0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
| jmp >7
}
|.code
}
} else if (opline->opcode == ZEND_SEND_FUNC_ARG) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 0)) {
return 0;
}
return 1;
}
} else {
| test dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
| jnz >1
|.cold_code
|1:
if (!zend_jit_send_ref(Dst, opline, op_array, op1_info, 1)) {
return 0;
}
| jmp >7
|.code
}
}
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
|.cold_code
|1:
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
| SET_ZVAL_TYPE_INFO arg_addr, IS_NULL
| test r0, r0
| jz ->exception_handler
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| jmp >7
|.code
} else {
|7:
return 1;
}
}
if (opline->opcode == ZEND_SEND_VAR_NO_REF) {
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R1, ZREG_R2
if (op1_info & MAY_BE_REF) {
| cmp cl, IS_REFERENCE
| je >7
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| jmp &exit_addr
} else {
| SET_EX_OPLINE opline, r0
| LOAD_ZVAL_ADDR FCARG1a, arg_addr
| EXT_CALL zend_jit_only_vars_by_reference, r0
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
} else {
if (op1_info & MAY_BE_REF) {
if (opline->op1_type == IS_CV) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, val_addr, op1_info, ZREG_R0, ZREG_R2
| TRY_ADDREF op1_info, ah, r2
} else {
zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 8);
| IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
|.cold_code
|1:
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR FCARG1a, op1_addr
| // ZVAL_COPY_VALUE(return_value, &ref->value);
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R0, ZREG_R2
| GC_DELREF FCARG1a
| je >1
| IF_NOT_REFCOUNTED ah, >2
| GC_ADDREF r2
| jmp >2
|1:
| EFREE_REG_REFERENCE
| jmp >2
|.code
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
|2:
}
} else {
if (op1_addr != op1_def_addr) {
if (!zend_jit_update_regs(Dst, opline->op1.var, op1_addr, op1_def_addr, op1_info)) {
return 0;
}
if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) {
op1_addr= op1_def_addr;
}
}
| ZVAL_COPY_VALUE arg_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
if (opline->op1_type == IS_CV) {
| TRY_ADDREF op1_info, ah, r2
}
}
}
|7:
return 1;
}
static int zend_jit_check_func_arg(dasm_State **Dst, const zend_op *opline)
{
uint32_t arg_num = opline->op2.num;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->call
&& JIT_G(current_frame)->call->func) {
if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) {
if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call);
| // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
|| if (reuse_ip) {
| or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
|| } else {
| mov r0, EX->call
| or dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
|| }
}
} else {
if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) {
TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call);
| // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
|| if (reuse_ip) {
| and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF
|| } else {
| mov r0, EX->call
| and dword [r0 + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF
|| }
}
}
} else {
// if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) {
uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2);
if (!zend_jit_reuse_ip(Dst)) {
return 0;
}
| mov r0, EX:RX->func
| test dword [r0 + offsetof(zend_function, quick_arg_flags)], mask
| jnz >1
|.cold_code
|1:
| // ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
| or dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ZEND_CALL_SEND_ARG_BY_REF
| jmp >1
|.code
| // ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF);
| and dword [RX + offsetof(zend_execute_data, This.u1.type_info)], ~ZEND_CALL_SEND_ARG_BY_REF
|1:
}
return 1;
}
static int zend_jit_smart_true(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
if (jmp) {
| jmp >7
}
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
if (jmp) {
| jmp >7
}
}
return 1;
}
static int zend_jit_smart_false(dasm_State **Dst, const zend_op *opline, int jmp, uint8_t smart_branch_opcode, uint32_t target_label)
{
if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
if (jmp) {
| jmp >7
}
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
if (jmp) {
| jmp >7
}
}
return 1;
}
static int zend_jit_defined(dasm_State **Dst, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t defined_label = (uint32_t)-1;
uint32_t undefined_label = (uint32_t)-1;
zval *zv = RT_CONSTANT(opline, opline->op1);
zend_jit_addr res_addr = 0;
if (smart_branch_opcode && !exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
undefined_label = target_label;
} else if (smart_branch_opcode == ZEND_JMPNZ) {
defined_label = target_label;
} else {
ZEND_UNREACHABLE();
}
}
| // if (CACHED_PTR(opline->extended_value)) {
| mov r0, EX->run_time_cache
| mov r0, aword [r0 + opline->extended_value]
| test r0, r0
| jz >1
| test r0, 0x1
| jnz >4
|.cold_code
|4:
| MEM_LOAD_ZTS FCARG1a, aword, executor_globals, zend_constants, FCARG1a
| shr r0, 1
| cmp dword [FCARG1a + offsetof(HashTable, nNumOfElements)], eax
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jz &exit_addr
} else {
| jz >3
}
} else if (undefined_label != (uint32_t)-1) {
| jz =>undefined_label
} else {
| jz >3
}
} else {
| jz >2
}
|1:
| SET_EX_OPLINE opline, r0
| LOAD_ADDR FCARG1a, zv
| EXT_CALL zend_jit_check_constant, r0
| test r0, r0
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jz >3
} else {
| jnz >3
}
| jmp &exit_addr
} else if (smart_branch_opcode) {
if (undefined_label != (uint32_t)-1) {
| jz =>undefined_label
} else {
| jz >3
}
if (defined_label != (uint32_t)-1) {
| jmp =>defined_label
} else {
| jmp >3
}
} else {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| jnz >1
|2:
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
| jmp >3
}
|.code
if (smart_branch_opcode) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (defined_label != (uint32_t)-1) {
| jmp =>defined_label
}
} else {
|1:
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
|3:
return 1;
}
static int zend_jit_type_check(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
uint32_t mask;
zend_jit_addr op1_addr = OP1_ADDR();
// TODO: support for is_resource() ???
ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE);
if (op1_info & MAY_BE_UNDEF) {
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
| IF_ZVAL_TYPE op1_addr, IS_UNDEF, >1
|.cold_code
|1:
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
zend_jit_check_exception_undef_result(Dst, opline);
if (opline->extended_value & MAY_BE_NULL) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
| jmp >7
}
} else if (!zend_jit_smart_true(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
} else {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0) {
| jmp >7
}
} else if (!zend_jit_smart_false(Dst, opline, (op1_info & (MAY_BE_ANY|MAY_BE_REF)) != 0, smart_branch_opcode, target_label)) {
return 0;
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
|.code
}
}
if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) {
mask = opline->extended_value;
if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
}
} else if (!zend_jit_smart_true(Dst, opline, 0, smart_branch_opcode, target_label, target_label2)) {
return 0;
}
} else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (!zend_jit_smart_false(Dst, opline, 0, smart_branch_opcode, target_label)) {
return 0;
}
} else {
bool invert = 0;
uint8_t type;
switch (mask) {
case MAY_BE_NULL: type = IS_NULL; break;
case MAY_BE_FALSE: type = IS_FALSE; break;
case MAY_BE_TRUE: type = IS_TRUE; break;
case MAY_BE_LONG: type = IS_LONG; break;
case MAY_BE_DOUBLE: type = IS_DOUBLE; break;
case MAY_BE_STRING: type = IS_STRING; break;
case MAY_BE_ARRAY: type = IS_ARRAY; break;
case MAY_BE_OBJECT: type = IS_OBJECT; break;
case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break;
case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break;
case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break;
case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break;
case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break;
case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break;
case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break;
default:
type = 0;
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR r0, op1_addr
| ZVAL_DEREF r0, op1_info
}
if (type == 0) {
if (smart_branch_opcode &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
| IF_ZVAL_REFCOUNTED op1_addr, >1
|.cold_code
|1:
}
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
| jnz >3
}
if (op1_info & MAY_BE_REF) {
| mov al, byte [r0 + 8]
} else {
| mov al, byte [FP + opline->op1.var + 8]
}
| mov byte T1, al // save
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op1_info, opline
| mov cl, byte T1 // restore
|jmp >2
}
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (!RC_MAY_BE_1(op1_info)) {
| jmp >3
}
|.code
}
|3:
if (op1_info & MAY_BE_REF) {
| mov cl, byte [r0 + 8]
} else {
| mov cl, byte [FP + opline->op1.var + 8]
}
|2:
} else {
if (op1_info & MAY_BE_REF) {
| mov cl, byte [r0 + 8]
} else {
| mov cl, byte [FP + opline->op1.var + 8]
}
}
| mov eax, 1
| shl eax, cl
| test eax, mask
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jne &exit_addr
} else {
| je &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| je =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jne =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
| setne al
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
} else {
if (smart_branch_opcode &&
(opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (Z_REFCOUNTED_P(cv)) {
| IF_ZVAL_REFCOUNTED op1_addr, >1
|.cold_code
|1:
}
| // if (!Z_DELREF_P(cv)) {
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
| jnz >3
}
if (op1_info & MAY_BE_REF) {
| mov al, byte [r0 + 8]
} else {
| mov al, byte [FP + opline->op1.var + 8]
}
| mov byte T1, al // save
| // zval_dtor_func(r);
| ZVAL_DTOR_FUNC op1_info, opline
| mov cl, byte T1 // restore
|jmp >2
}
if ((op1_info) & (MAY_BE_ANY-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (!RC_MAY_BE_1(op1_info)) {
| jmp >3
}
|.code
}
|3:
if (op1_info & MAY_BE_REF) {
| mov cl, byte [r0 + 8]
} else {
| mov cl, byte [FP + opline->op1.var + 8]
}
|2:
| cmp cl, type
} else {
if (op1_info & MAY_BE_REF) {
| cmp byte [r0 + 8], type
} else {
| cmp byte [FP + opline->op1.var + 8], type
}
}
if (exit_addr) {
if (invert) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jne &exit_addr
} else {
| je &exit_addr
}
} else {
if (smart_branch_opcode == ZEND_JMPNZ) {
| je &exit_addr
} else {
| jne &exit_addr
}
}
} else if (smart_branch_opcode) {
if (invert) {
if (smart_branch_opcode == ZEND_JMPZ) {
| je =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jne =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
if (smart_branch_opcode == ZEND_JMPZ) {
| jne =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| je =>target_label
} else {
ZEND_UNREACHABLE();
}
}
} else {
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (invert) {
| setne al
} else {
| sete al
}
| movzx eax, al
| add eax, 2
| SET_ZVAL_TYPE_INFO res_addr, eax
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
}
}
}
|7:
return 1;
}
static int zend_jit_leave_frame(dasm_State **Dst)
{
| // EG(current_execute_data) = EX(prev_execute_data);
| mov r0, EX->prev_execute_data
| MEM_STORE_ZTS aword, executor_globals, current_execute_data, r0, r2
return 1;
}
static int zend_jit_free_cvs(dasm_State **Dst)
{
| // EG(current_execute_data) = EX(prev_execute_data);
| mov FCARG1a, EX->prev_execute_data
| MEM_STORE_ZTS aword, executor_globals, current_execute_data, FCARG1a, r0
| // zend_free_compiled_variables(execute_data);
| mov FCARG1a, FP
| EXT_CALL zend_free_compiled_variables, r0
return 1;
}
static int zend_jit_free_cv(dasm_State **Dst, uint32_t info, uint32_t var)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
uint32_t offset = EX_NUM_TO_VAR(var);
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, offset), info, 1, 1, NULL
}
return 1;
}
static int zend_jit_free_op(dasm_State **Dst, const zend_op *opline, uint32_t info, uint32_t var_offset)
{
if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
| ZVAL_PTR_DTOR ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, 1, opline
}
return 1;
}
static int zend_jit_leave_func(dasm_State **Dst,
const zend_op_array *op_array,
const zend_op *opline,
uint32_t op1_info,
bool left_frame,
zend_jit_trace_rec *trace,
zend_jit_trace_info *trace_info,
int indirect_var_access,
int may_throw)
{
bool may_be_top_frame =
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_IS_NESTED(JIT_G(current_frame));
bool may_need_call_helper =
indirect_var_access || /* may have symbol table */
!op_array->function_name || /* may have symbol table */
may_be_top_frame ||
(op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */
JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */
(uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */
bool may_need_release_this =
!(op_array->fn_flags & ZEND_ACC_CLOSURE) &&
op_array->scope &&
!(op_array->fn_flags & ZEND_ACC_STATIC) &&
(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
!JIT_G(current_frame) ||
!TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame)));
if (may_need_release_this) {
| mov FCARG1d, dword [FP + offsetof(zend_execute_data, This.u1.type_info)]
}
if (may_need_call_helper) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
/* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */
if (may_need_release_this) {
| test FCARG1d, (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE)
} else {
| test dword [FP + offsetof(zend_execute_data, This.u1.type_info)], (ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE)
}
if (trace && trace->op != ZEND_JIT_TRACE_END) {
| jnz >1
|.cold_code
|1:
if (!GCC_GLOBAL_REGS) {
| mov FCARG1a, FP
}
| EXT_CALL zend_jit_leave_func_helper, r0
if (may_be_top_frame) {
// TODO: try to avoid this check ???
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
#if 0
/* this check should be handled by the following OPLINE guard */
| cmp IP, zend_jit_halt_op
| je ->trace_halt
#endif
} else if (GCC_GLOBAL_REGS) {
| test IP, IP
| je ->trace_halt
} else {
| test eax, eax
| jl ->trace_halt
}
}
if (!GCC_GLOBAL_REGS) {
| // execute_data = EG(current_execute_data)
| MEM_LOAD_ZTS FP, aword, executor_globals, current_execute_data, r0
}
| jmp >8
|.code
} else {
| jnz ->leave_function_handler
}
}
if (op_array->fn_flags & ZEND_ACC_CLOSURE) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
| // OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func)));
| mov FCARG1a, EX->func
| sub FCARG1a, sizeof(zend_object)
| OBJ_RELEASE ZREG_FCARG1, >4
|4:
} else if (may_need_release_this) {
if (!left_frame) {
left_frame = 1;
if (!zend_jit_leave_frame(Dst)) {
return 0;
}
}
if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) {
| // if (call_info & ZEND_CALL_RELEASE_THIS)
| test FCARG1d, ZEND_CALL_RELEASE_THIS
| je >4
}
| // zend_object *object = Z_OBJ(execute_data->This);
| mov FCARG1a, EX->This.value.obj
| // OBJ_RELEASE(object);
| OBJ_RELEASE ZREG_FCARG1, >4
|4:
// TODO: avoid EG(excption) check for $this->foo() calls
may_throw = 1;
}
| // EG(vm_stack_top) = (zval*)execute_data;
| MEM_STORE_ZTS aword, executor_globals, vm_stack_top, FP, r0
| // execute_data = EX(prev_execute_data);
| mov FP, EX->prev_execute_data
if (!left_frame) {
| // EG(current_execute_data) = execute_data;
| MEM_STORE_ZTS aword, executor_globals, current_execute_data, FP, r0
}
|9:
if (trace) {
if (trace->op != ZEND_JIT_TRACE_END
&& (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
zend_jit_reset_last_valid_opline();
} else {
| LOAD_IP
| ADD_IP sizeof(zend_op)
}
|8:
if (trace->op == ZEND_JIT_TRACE_BACK
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) {
const zend_op *next_opline = trace->opline;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) {
/* exception might be thrown during destruction of unused return value */
| // if (EG(exception))
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| jne ->leave_throw_handler
}
do {
trace++;
} while (trace->op == ZEND_JIT_TRACE_INIT_CALL);
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
next_opline = trace->opline;
ZEND_ASSERT(next_opline != NULL);
if (trace->op == ZEND_JIT_TRACE_END
&& trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) {
trace_info->flags |= ZEND_JIT_TRACE_LOOP;
| CMP_IP next_opline
| je =>0 // LOOP
#ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE
| JMP_IP
#else
| jmp ->trace_escape
#endif
} else {
| CMP_IP next_opline
| jne ->trace_escape
}
zend_jit_set_last_valid_opline(trace->opline);
return 1;
} else if (may_throw ||
(((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& (op1_info & MAY_BE_RC1)
&& (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY)))
&& (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) {
| // if (EG(exception))
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| jne ->leave_throw_handler
}
return 1;
} else {
| // if (EG(exception))
| MEM_CMP_ZTS aword, executor_globals, exception, 0, r0
| LOAD_IP
| jne ->leave_throw_handler
| // opline = EX(opline) + 1
| ADD_IP sizeof(zend_op)
}
if (zend_jit_vm_kind == ZEND_VM_KIND_HYBRID) {
| ADD_HYBRID_SPAD
#ifdef CONTEXT_THREADED_JIT
| push aword [IP]
| ret
#else
| JMP_IP
#endif
} else if (GCC_GLOBAL_REGS) {
| add r4, SPAD // stack alignment
#ifdef CONTEXT_THREADED_JIT
| push aword [IP]
| ret
#else
| JMP_IP
#endif
} else {
#ifdef CONTEXT_THREADED_JIT
ZEND_UNREACHABLE();
// TODO: context threading can't work without GLOBAL REGS because we have to change
// the value of execute_data in execute_ex()
| mov FCARG1a, FP
| mov r0, aword [FP]
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| push aword [r0]
| ret
#else
| mov FP, aword T2 // restore FP
| mov RX, aword T3 // restore IP
| add r4, NR_SPAD // stack alignment
| mov r0, 2 // ZEND_VM_LEAVE
| ret
#endif
}
return 1;
}
static int zend_jit_return(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr)
{
zend_jit_addr ret_addr;
int8_t return_value_used;
ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name);
ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF));
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) {
if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) {
return_value_used = 1;
} else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) {
return_value_used = 0;
} else {
return_value_used = -1;
}
} else {
return_value_used = -1;
}
if (ZEND_OBSERVER_ENABLED) {
if (Z_MODE(op1_addr) == IS_REG) {
zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!zend_jit_spill_store(Dst, op1_addr, dst, op1_info, 1)) {
return 0;
}
op1_addr = dst;
}
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
| mov FCARG1a, FP
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_observer_fcall_end, r0
}
// if (!EX(return_value))
if (Z_MODE(op1_addr) == IS_REG && Z_REG(op1_addr) == ZREG_R1) {
if (return_value_used != 0) {
| mov r2, EX->return_value
}
if (return_value_used == -1) {
| test r2, r2
}
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R2, 0);
} else {
if (return_value_used != 0) {
| mov r1, EX->return_value
}
if (return_value_used == -1) {
| test r1, r1
}
ret_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R1, 0);
}
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (return_value_used == -1) {
| jz >1
|.cold_code
|1:
}
if (return_value_used != 1) {
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (jit_return_label >= 0) {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, =>jit_return_label
} else {
| IF_NOT_ZVAL_REFCOUNTED op1_addr, >9
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
| GC_DELREF FCARG1a
if (RC_MAY_BE_1(op1_info)) {
if (RC_MAY_BE_N(op1_info)) {
if (jit_return_label >= 0) {
| jnz =>jit_return_label
} else {
| jnz >9
}
}
| //SAVE_OPLINE()
| ZVAL_DTOR_FUNC op1_info, opline
| //????mov r1, EX->return_value // reload ???
}
if (return_value_used == -1) {
if (jit_return_label >= 0) {
| jmp =>jit_return_label
} else {
| jmp >9
}
|.code
}
}
} else if (return_value_used == -1) {
if (jit_return_label >= 0) {
| jz =>jit_return_label
} else {
| jz >9
}
}
if (return_value_used == 0) {
|9:
return 1;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else if (opline->op1_type == IS_TMP_VAR) {
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
} else if (opline->op1_type == IS_CV) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR r0, op1_addr
| ZVAL_DEREF r0, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
}
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) ||
!op_array->function_name) {
| TRY_ADDREF op1_info, ah, r2
} else if (return_value_used != 1) {
| // if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr);
| SET_ZVAL_TYPE_INFO op1_addr, IS_NULL
}
}
} else {
if (op1_info & MAY_BE_REF) {
zend_jit_addr ref_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, offsetof(zend_reference, val));
| IF_ZVAL_TYPE op1_addr, IS_REFERENCE, >1
|.cold_code
|1:
| // zend_refcounted *ref = Z_COUNTED_P(retval_ptr);
| GET_ZVAL_PTR r0, op1_addr
| // ZVAL_COPY_VALUE(return_value, &ref->value);
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, ref_addr, op1_info, ZREG_R2, ZREG_R2
| GC_DELREF r0
| je >2
| // if (IS_REFCOUNTED())
if (jit_return_label >= 0) {
| IF_NOT_REFCOUNTED dh, =>jit_return_label
} else {
| IF_NOT_REFCOUNTED dh, >9
}
| // ADDREF
| GET_ZVAL_PTR r2, ret_addr // reload
| GC_ADDREF r2
if (jit_return_label >= 0) {
| jmp =>jit_return_label
} else {
| jmp >9
}
|2:
| EFREE_REFERENCE r0
if (jit_return_label >= 0) {
| jmp =>jit_return_label
} else {
| jmp >9
}
|.code
}
| ZVAL_COPY_VALUE ret_addr, MAY_BE_ANY, op1_addr, op1_info, ZREG_R0, ZREG_R2
}
|9:
return 1;
}
static int zend_jit_zval_copy_deref(dasm_State **Dst, zend_jit_addr res_addr, zend_jit_addr val_addr, zend_reg type_reg)
{
ZEND_ASSERT(type_reg == ZREG_R2);
|.if not(X64)
|| if (Z_REG(val_addr) == ZREG_R1) {
| GET_ZVAL_W2 r0, val_addr
|| }
|.endif
| GET_ZVAL_PTR r1, val_addr
|.if not(X64)
|| if (Z_REG(val_addr) != ZREG_R1) {
| GET_ZVAL_W2 r0, val_addr
|| }
|.endif
| IF_NOT_REFCOUNTED dh, >2
| IF_NOT_TYPE dl, IS_REFERENCE, >1
| GET_Z_TYPE_INFO edx, r1+offsetof(zend_reference, val)
|.if not(X64)
| GET_Z_W2 r0, r1+offsetof(zend_reference, val)
|.endif
| GET_Z_PTR r1, r1+offsetof(zend_reference, val)
| IF_NOT_REFCOUNTED dh, >2
|1:
| GC_ADDREF r1
|2:
| SET_ZVAL_PTR res_addr, r1
|.if not(X64)
| SET_ZVAL_W2 res_addr, r0
|.endif
| SET_ZVAL_TYPE_INFO res_addr, edx
return 1;
}
static int zend_jit_fetch_dim_read(dasm_State **Dst,
const zend_op *opline,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
uint32_t res_info,
zend_jit_addr res_addr,
uint8_t dim_type)
{
zend_jit_addr orig_op1_addr, op2_addr;
const void *exit_addr = NULL;
const void *not_found_exit_addr = NULL;
const void *res_exit_addr = NULL;
bool result_avoid_refcounting = 0;
uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0;
int may_throw = 0;
orig_op1_addr = OP1_ADDR();
op2_addr = OP2_ADDR();
if (opline->opcode != ZEND_FETCH_DIM_IS
&& JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if ((res_info & MAY_BE_GUARD)
&& JIT_G(current_frame)
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) {
uint32_t flags = 0;
uint32_t old_op1_info = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
if (opline->opcode != ZEND_FETCH_LIST_R
&& (opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !op1_avoid_refcounting) {
flags |= ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->op2_type & (IS_VAR|IS_TMP_VAR))
&& (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
flags |= ZEND_JIT_EXIT_FREE_OP2;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
if (op1_avoid_refcounting) {
old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var));
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
res_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!res_exit_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
}
if (opline->opcode == ZEND_FETCH_DIM_IS
&& !(res_info & MAY_BE_NULL)) {
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NULL);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!not_found_exit_addr) {
return 0;
}
}
if (op1_avoid_refcounting) {
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info);
}
}
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
}
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr
if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) ||
(opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, dim_type, res_exit_addr, not_found_exit_addr, exit_addr)) {
return 0;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) {
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) {
if (exit_addr && !(op1_info & MAY_BE_OBJECT)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
}
}
| SET_EX_OPLINE opline, r0
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr
if (opline->opcode != ZEND_FETCH_DIM_IS) {
if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) {
| GET_ZVAL_LVAL ZREG_FCARG2, op2_addr
| EXT_CALL zend_jit_fetch_dim_str_offset_r_helper, r0
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
| EXT_CALL zend_jit_fetch_dim_str_r_helper, r0
}
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_fetch_dim_str_is_helper, r0
|.if not(X64)
| add r4, 12
|.endif
}
if ((op1_info & MAY_BE_ARRAY) ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING)))) {
| jmp >9 // END
}
|6:
}
if (op1_info & MAY_BE_OBJECT) {
may_throw = 1;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) {
if (exit_addr) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >6
}
}
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
if (opline->opcode != ZEND_FETCH_DIM_IS) {
| EXT_CALL zend_jit_fetch_dim_obj_r_helper, r0
} else {
| EXT_CALL zend_jit_fetch_dim_obj_is_helper, r0
}
|.if not(X64)
| add r4, 12
|.endif
if ((op1_info & MAY_BE_ARRAY) ||
(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
| jmp >9 // END
}
|6:
}
if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))
&& (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) {
if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) {
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) {
may_throw = 1;
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
if (op2_info & MAY_BE_UNDEF) {
may_throw = 1;
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
}
if (opline->opcode != ZEND_FETCH_DIM_IS && opline->opcode != ZEND_FETCH_LIST_R) {
may_throw = 1;
if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) {
| LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr
} else {
| SET_EX_OPLINE opline, r0
if (Z_MODE(op1_addr) != IS_MEM_ZVAL ||
Z_REG(op1_addr) != ZREG_FCARG1 ||
Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
}
| EXT_CALL zend_jit_invalid_array_access, r0
}
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
if (op1_info & MAY_BE_ARRAY) {
| jmp >9 // END
}
}
if (op1_info & MAY_BE_ARRAY) {
|.code
}
}
if (op1_info & MAY_BE_ARRAY) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
|8:
if (res_exit_addr) {
uint8_t type = concrete_type(res_info);
if ((op1_info & MAY_BE_ARRAY_OF_REF)
&& dim_type != IS_UNKNOWN
&& dim_type != IS_REFERENCE) {
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE val_addr, type, >1
|.cold_code
|1:
| IF_NOT_ZVAL_TYPE val_addr, IS_REFERENCE, &res_exit_addr
| GET_Z_PTR r0, r0
| add r0, offsetof(zend_reference, val)
| IF_ZVAL_TYPE val_addr, type, >1
| jmp &res_exit_addr
|.code
|1:
} else {
| GET_ZVAL_TYPE_INFO edx, val_addr
| IF_NOT_TYPE dl, type, >1
|.cold_code
|1:
| IF_NOT_TYPE dl, IS_REFERENCE, &res_exit_addr
| GET_Z_PTR r0, r0
| add r0, offsetof(zend_reference, val)
| GET_ZVAL_TYPE_INFO edx, val_addr
| IF_TYPE dl, type, >1
| jmp &res_exit_addr
|.code
|1:
}
} else {
if (op1_info & MAY_BE_ARRAY_OF_REF) {
| ZVAL_DEREF r0, MAY_BE_REF
}
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE val_addr, type, &res_exit_addr
} else {
| GET_ZVAL_TYPE_INFO edx, val_addr
| IF_NOT_TYPE dl, type, &res_exit_addr
}
}
| // ZVAL_COPY
|7:
| ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
if (type < IS_STRING) {
if (Z_REG(res_addr) != ZREG_FP ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
| SET_ZVAL_TYPE_INFO res_addr, type
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, edx
if (!result_avoid_refcounting) {
| TRY_ADDREF res_info, dh, r1
}
}
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
} else if (op1_info & MAY_BE_ARRAY_OF_REF) {
| // ZVAL_COPY_DEREF
| GET_ZVAL_TYPE_INFO Rd(ZREG_R2), val_addr
if (!zend_jit_zval_copy_deref(Dst, res_addr, val_addr, ZREG_R2)) {
return 0;
}
} else {
| // ZVAL_COPY
| ZVAL_COPY_VALUE res_addr, -1, val_addr, res_info, ZREG_R1, ZREG_R2
| TRY_ADDREF res_info, ch, r2
}
}
|9: // END
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetGet() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) {
if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
}
if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) {
if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_fetch_dim(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t op2_info,
zend_jit_addr res_addr,
uint8_t dim_type)
{
zend_jit_addr op2_addr;
int may_throw = 0;
op2_addr = (opline->op2_type != IS_UNUSED) ? OP2_ADDR() : 0;
if (opline->opcode == ZEND_FETCH_DIM_RW) {
| SET_EX_OPLINE opline, r0
}
if (op1_info & MAY_BE_REF) {
may_throw = 1;
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_REFERENCE, >1
| GET_Z_PTR FCARG2a, FCARG1a
| IF_NOT_TYPE byte [FCARG2a + offsetof(zend_reference, val) + offsetof(zval, u1.v.type)], IS_ARRAY, >2
| lea FCARG1a, [FCARG2a + offsetof(zend_reference, val)]
| jmp >3
|.cold_code
|2:
if (opline->opcode != ZEND_FETCH_DIM_RW) {
| SET_EX_OPLINE opline, r0
}
| EXT_CALL zend_jit_prepare_assign_dim_ref, r0
| test r0, r0
| mov FCARG1a, r0
| jne >1
| jmp ->exception_handler_undef
|.code
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
|3:
| SEPARATE_ARRAY op1_addr, op1_info, 1
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
| CMP_ZVAL_TYPE op1_addr, IS_NULL
| jg >7
}
if (Z_REG(op1_addr) != ZREG_FP) {
| mov T1, Ra(Z_REG(op1_addr)) // save
}
if ((op1_info & MAY_BE_UNDEF)
&& opline->opcode == ZEND_FETCH_DIM_RW) {
may_throw = 1;
if (op1_info & MAY_BE_NULL) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
| // ZVAL_ARR(container, zend_new_array(8));
| EXT_CALL _zend_new_array_0, r0
if (Z_REG(op1_addr) != ZREG_FP) {
| mov Ra(Z_REG(op1_addr)), T1 // restore
}
| SET_ZVAL_LVAL op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_ARRAY_EX
| mov FCARG1a, r0
if (op1_info & MAY_BE_ARRAY) {
| jmp >1
|.code
|1:
}
}
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|6:
if (opline->op2_type == IS_UNUSED) {
may_throw = 1;
| // var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval));
| LOAD_ADDR_ZTS FCARG2a, executor_globals, uninitialized_zval
| EXT_CALL zend_hash_next_index_insert, r0
| // if (UNEXPECTED(!var_ptr)) {
| test r0, r0
| jz >1
|.cold_code
|1:
| // zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied");
| CANNOT_ADD_ELEMENT opline
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF
| //ZEND_VM_C_GOTO(assign_dim_op_ret_null);
| jmp >8
|.code
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT
} else {
uint32_t type;
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
type = BP_VAR_W;
break;
case ZEND_FETCH_DIM_RW:
may_throw = 1;
type = BP_VAR_RW;
break;
case ZEND_FETCH_DIM_UNSET:
type = BP_VAR_UNSET;
break;
default:
ZEND_UNREACHABLE();
}
if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) {
may_throw = 1;
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, dim_type, NULL, NULL, NULL)) {
return 0;
}
|8:
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT
if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) {
|.cold_code
|9:
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >8
|.code
}
}
}
if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) {
may_throw = 1;
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
|.cold_code
|7:
}
if (opline->opcode != ZEND_FETCH_DIM_RW) {
| SET_EX_OPLINE opline, r0
}
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
switch (opline->opcode) {
case ZEND_FETCH_DIM_W:
case ZEND_FETCH_LIST_W:
| EXT_CALL zend_jit_fetch_dim_obj_w_helper, r0
break;
case ZEND_FETCH_DIM_RW:
| EXT_CALL zend_jit_fetch_dim_obj_rw_helper, r0
break;
// case ZEND_FETCH_DIM_UNSET:
// | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0
// break;
default:
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) {
| jmp >8 // END
|.code
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) {
/* ASSIGN_DIM may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR))
&& (op2_info & MAY_HAVE_DTOR)
&& (op2_info & MAY_BE_RC1)) {
may_throw = 1;
}
|8:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_isset_isempty_dim(dasm_State **Dst,
const zend_op *opline,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_avoid_refcounting,
uint32_t op2_info,
uint8_t dim_type,
int may_throw,
uint8_t smart_branch_opcode,
uint32_t target_label,
uint32_t target_label2,
const void *exit_addr)
{
zend_jit_addr op2_addr, res_addr;
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
op2_addr = OP2_ADDR();
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_ARRAY) {
const void *found_exit_addr = NULL;
const void *not_found_exit_addr = NULL;
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
| GET_ZVAL_LVAL ZREG_FCARG1, op1_addr
if (exit_addr
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY))
&& !may_throw
&& (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting)
&& (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) {
if (smart_branch_opcode == ZEND_JMPNZ) {
found_exit_addr = exit_addr;
} else {
not_found_exit_addr = exit_addr;
}
}
if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, dim_type, found_exit_addr, not_found_exit_addr, NULL)) {
return 0;
}
if (found_exit_addr) {
|9:
return 1;
} else if (not_found_exit_addr) {
|8:
return 1;
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) {
if (op1_info & MAY_BE_ARRAY) {
|.cold_code
|7:
}
if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) {
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) {
ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL);
| LOAD_ADDR FCARG2a, (Z_ZV(op2_addr) + 1)
} else {
| LOAD_ZVAL_ADDR FCARG2a, op2_addr
}
| EXT_CALL zend_jit_isset_dim_helper, r0
| test r0, r0
| jz >9
if (op1_info & MAY_BE_ARRAY) {
| jmp >8
|.code
}
} else {
if (op2_info & MAY_BE_UNDEF) {
if (op2_info & MAY_BE_ANY) {
| IF_NOT_ZVAL_TYPE op2_addr, IS_UNDEF, >1
}
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op2.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
}
if (op1_info & MAY_BE_ARRAY) {
| jmp >9
|.code
}
}
}
#ifdef ZEND_JIT_USE_RC_INFERENCE
if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) {
/* Magic offsetExists() may increase refcount of the key */
op2_info |= MAY_BE_RCN;
}
#endif
if (op1_info & (MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_OBJECT)) {
|8:
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (!op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp &exit_addr
} else {
| jmp >8
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label2
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
| jmp >8
}
} else {
| NIY // TODO: support for empty()
}
}
|9: // not found
| FREE_OP opline->op2_type, opline->op2, op2_info, 0, opline
if (!op1_avoid_refcounting) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
if (may_throw) {
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (!(opline->extended_value & ZEND_ISEMPTY)) {
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jmp =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
} else {
ZEND_UNREACHABLE();
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
} else {
| NIY // TODO: support for empty()
}
|8:
return 1;
}
static int zend_jit_bind_global(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr op1_addr = OP1_ADDR();
zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2));
| // idx = (uint32_t)(uintptr_t)CACHED_PTR(opline->extended_value) - 1;
| mov FCARG2a, EX->run_time_cache
| mov r0, aword [FCARG2a + opline->extended_value]
| sub r0, 1
| // if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket)))
| MEM_LOAD_ZTS ecx, dword, executor_globals, symbol_table.nNumUsed, r1
|.if X64
| shl r1, 5
|.else
| imul r1, sizeof(Bucket)
|.endif
| cmp r0, r1
| jae >9
| // Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx);
| MEM_LOAD_OP_ZTS add, r0, aword, executor_globals, symbol_table.arData, r1
| IF_NOT_Z_TYPE r0, IS_REFERENCE, >9
| // (EXPECTED(p->key == varname))
| ADDR_CMP aword [r0 + offsetof(Bucket, key)], varname, r1
| jne >9
| GET_Z_PTR r0, r0
| GC_ADDREF r0
|1:
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr)))
| IF_ZVAL_REFCOUNTED op1_addr, >2
|.cold_code
|2:
}
| // zend_refcounted *garbage = Z_COUNTED_P(variable_ptr);
| GET_ZVAL_PTR FCARG1a, op1_addr
| // ZVAL_REF(variable_ptr, ref)
| SET_ZVAL_PTR op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
| // if (GC_DELREF(garbage) == 0)
| GC_DELREF FCARG1a
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
| jnz >3
} else {
| jnz >5
}
| ZVAL_DTOR_FUNC op1_info, opline
| jmp >5
if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) {
|3:
| // GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr)
| IF_GC_MAY_NOT_LEAK FCARG1a, >5
if (opline) {
| SET_EX_OPLINE opline, r0
}
| EXT_CALL gc_possible_root, r1
| jmp >5
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
|.code
}
}
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| // ZVAL_REF(variable_ptr, ref)
| SET_ZVAL_PTR op1_addr, r0
| SET_ZVAL_TYPE_INFO op1_addr, IS_REFERENCE_EX
}
|5:
//END of handler
|.cold_code
|9:
| LOAD_ADDR FCARG1a, (ptrdiff_t)varname
if (opline->extended_value) {
| add FCARG2a, opline->extended_value
}
| EXT_CALL zend_jit_fetch_global_helper, r0
| jmp <1
|.code
return 1;
}
static int zend_jit_verify_arg_type(dasm_State **Dst, const zend_op *opline, zend_arg_info *arg_info, bool check_exception)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
bool in_cold = 0;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
zend_reg tmp_reg = (type_mask == 0 || is_power_of_two(type_mask)) ? ZREG_FCARG1 : ZREG_R0;
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& JIT_G(current_frame)->prev) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var));
if (type != IS_UNKNOWN && (type_mask & (1u << type))) {
return 1;
}
}
if (ZEND_ARG_SEND_MODE(arg_info)) {
if (opline->opcode == ZEND_RECV_INIT) {
| LOAD_ZVAL_ADDR Ra(tmp_reg), res_addr
| ZVAL_DEREF Ra(tmp_reg), MAY_BE_REF
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, 0);
} else {
| GET_ZVAL_PTR Ra(tmp_reg), res_addr
res_addr = ZEND_ADDR_MEM_ZVAL(tmp_reg, offsetof(zend_reference, val));
}
}
if (type_mask != 0) {
if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
| IF_NOT_ZVAL_TYPE res_addr, type_code, >1
} else {
| mov edx, 1
| mov cl, byte [Ra(Z_REG(res_addr))+Z_OFFSET(res_addr)+offsetof(zval, u1.v.type)]
| shl edx, cl
| test edx, type_mask
| je >1
}
|.cold_code
|1:
in_cold = 1;
}
if (Z_REG(res_addr) != ZREG_FCARG1 || Z_OFFSET(res_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, res_addr
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, r0
} else {
| ADDR_STORE aword EX->opline, opline, r0
}
| LOAD_ADDR FCARG2a, (ptrdiff_t)arg_info
| EXT_CALL zend_jit_verify_arg_slow, r0
if (check_exception) {
| test al, al
if (in_cold) {
| jnz >1
| jmp ->exception_handler
|.code
|1:
} else {
| jz ->exception_handler
}
} else if (in_cold) {
| jmp >1
|.code
|1:
}
return 1;
}
static int zend_jit_recv(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array)
{
uint32_t arg_num = opline->op1.num;
zend_arg_info *arg_info = NULL;
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
if (EXPECTED(arg_num <= op_array->num_args)) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) {
arg_info = &op_array->arg_info[op_array->num_args];
}
if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) {
arg_info = NULL;
}
}
if (arg_info || (opline+1)->opcode != ZEND_RECV) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 ||
arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| cmp dword EX->This.u2.num_args, arg_num
| jb &exit_addr
}
} else {
| cmp dword EX->This.u2.num_args, arg_num
| jb >1
|.cold_code
|1:
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, r0
} else {
| ADDR_STORE aword EX->opline, opline, r0
}
| mov FCARG1a, FP
| EXT_CALL zend_missing_arg_error, r0
| jmp ->exception_handler
|.code
}
}
if (arg_info) {
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, 1)) {
return 0;
}
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if ((opline+1)->opcode != ZEND_RECV && (opline+1)->opcode != ZEND_RECV_INIT) {
| LOAD_IP_ADDR (opline + 1)
zend_jit_set_last_valid_opline(opline + 1);
}
}
return 1;
}
static int zend_jit_recv_init(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw)
{
uint32_t arg_num = opline->op1.num;
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& JIT_G(current_frame)
&& TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) {
if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) {
| ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
}
} else {
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE ||
(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) {
| cmp dword EX->This.u2.num_args, arg_num
| jae >5
}
| ZVAL_COPY_CONST res_addr, -1, -1, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
}
if (Z_CONSTANT_P(zv)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
| SET_EX_OPLINE opline, r0
} else {
| ADDR_STORE aword EX->opline, opline, r0
}
| LOAD_ZVAL_ADDR FCARG1a, res_addr
| mov r0, EX->func
| mov FCARG2a, [r0 + offsetof(zend_op_array, scope)]
| .if X64
| EXT_CALL zval_update_constant_ex, r0
| .else
| EXT_CALL zval_update_constant_ex, r0
| .endif
| test al, al
| jnz >1
|.cold_code
|1:
| ZVAL_PTR_DTOR res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, 0, opline
| SET_ZVAL_TYPE_INFO res_addr, IS_UNDEF
| jmp ->exception_handler
|.code
}
|5:
if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) {
do {
zend_arg_info *arg_info;
if (arg_num <= op_array->num_args) {
arg_info = &op_array->arg_info[arg_num-1];
} else if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
arg_info = &op_array->arg_info[op_array->num_args];
} else {
break;
}
if (!ZEND_TYPE_IS_SET(arg_info->type)) {
break;
}
if (!zend_jit_verify_arg_type(Dst, opline, arg_info, may_throw)) {
return 0;
}
} while (0);
}
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (is_last) {
| LOAD_IP_ADDR (opline + 1)
zend_jit_set_last_valid_opline(opline + 1);
}
}
return 1;
}
static int zend_jit_class_guard(dasm_State **Dst, const zend_op *opline, zend_class_entry *ce)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
|.if X64
|| if (!IS_SIGNED_32BIT(ce)) {
| mov64 r0, ((ptrdiff_t)ce)
| cmp aword [FCARG1a + offsetof(zend_object, ce)], r0
|| } else {
| cmp aword [FCARG1a + offsetof(zend_object, ce)], ce
|| }
|.else
| cmp aword [FCARG1a + offsetof(zend_object, ce)], ce
|.endif
| jne &exit_addr
return 1;
}
static int zend_jit_fetch_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
bool op1_avoid_refcounting,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_property_info *prop_info;
bool may_be_dynamic = 1;
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
uint32_t res_info = RES_INFO();
bool type_loaded = 0;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >7
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >5
| mov r0, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*)]
may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array);
if (may_be_dynamic) {
| test r0, r0
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| jl >5
} else {
| jl >8 // dynamic property
}
}
| mov edx, dword [FCARG1a + r0 + 8]
| IF_UNDEF dl, >5
| add FCARG1a, r0
type_loaded = 1;
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
| mov r0, EX->run_time_cache
| mov FCARG2a, aword [r0 + (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2]
| test FCARG2a, FCARG2a
| jnz >1
|.cold_code
|1:
| test dword [FCARG2a + offsetof(zend_property_info, flags)], ZEND_ACC_READONLY
if (flags) {
| jz >3
} else {
| jz >4
}
| IF_NOT_Z_TYPE FCARG1a, IS_OBJECT, >2
| GET_Z_PTR r0, FCARG1a
| GC_ADDREF r0
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX
| jmp >9
|2:
| mov eax, dword [FCARG1a + offsetof(zval, u2.extra)]
| test eax, IS_PROP_REINITABLE
| jz >6
| and eax, ~IS_PROP_REINITABLE
| mov dword [FCARG1a + offsetof(zval, u2.extra)], eax
if (flags) {
| jmp >3
} else {
| jmp >4
}
|6:
| mov FCARG1a, FCARG2a
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_readonly_property_modification_error, r0
| SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR
| jmp >9
|3:
if (flags == ZEND_FETCH_DIM_WRITE) {
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_check_array_promotion, r0
| jmp >9
} else if (flags == ZEND_FETCH_REF) {
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_create_typed_ref, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
} else {
ZEND_ASSERT(flags == 0);
}
|.code
|4:
}
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) {
/* perform IS_UNDEF check only after result type guard (during deoptimization) */
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
type_loaded = 1;
| mov edx, dword [FCARG1a + prop_info->offset + 8]
| IF_UNDEF dl, &exit_addr
}
} else {
type_loaded = 1;
| mov edx, dword [FCARG1a + prop_info->offset + 8]
| IF_UNDEF dl, >5
}
if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) {
if (!type_loaded) {
type_loaded = 1;
| mov edx, dword [FCARG1a + prop_info->offset + offsetof(zval, u1.type_info)]
}
| IF_NOT_TYPE dl, IS_OBJECT, >4
| GET_ZVAL_PTR r0, prop_addr
| GC_ADDREF r0
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_OBJECT_EX
| jmp >9
|.cold_code
|4:
| mov eax, dword [FCARG1a + prop_info->offset + offsetof(zval, u2.extra)]
| test eax, IS_PROP_REINITABLE
| jz >6
| and eax, ~IS_PROP_REINITABLE
| mov dword [FCARG1a + prop_info->offset + offsetof(zval, u2.extra)], eax
| jmp >4
|6:
| LOAD_ADDR FCARG1a, prop_info
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_readonly_property_modification_error, r0
| SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR
| jmp >9
|.code
|4:
}
if (opline->opcode == ZEND_FETCH_OBJ_W
&& (opline->extended_value & ZEND_FETCH_OBJ_FLAGS)
&& ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS;
if (flags == ZEND_FETCH_DIM_WRITE) {
if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) {
if (!type_loaded) {
type_loaded = 1;
| mov edx, dword [FCARG1a + prop_info->offset + 8]
}
| cmp dl, IS_FALSE
| jle >1
|.cold_code
|1:
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
| LOAD_ADDR FCARG2a, prop_info
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_check_array_promotion, r0
| jmp >9
|.code
}
} else if (flags == ZEND_FETCH_REF) {
if (!type_loaded) {
type_loaded = 1;
| mov edx, dword [FCARG1a + prop_info->offset + 8]
}
| IF_TYPE dl, IS_REFERENCE, >1
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
| EXT_CALL zend_jit_create_typed_ref, r0
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
|1:
} else {
ZEND_UNREACHABLE();
}
}
}
if (opline->opcode == ZEND_FETCH_OBJ_W) {
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
| SET_ZVAL_PTR res_addr, FCARG1a
| SET_ZVAL_TYPE_INFO res_addr, IS_INDIRECT
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) {
ssa->var_info[ssa_op->result_def].indirect_reference = 1;
}
} else {
bool result_avoid_refcounting = 0;
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) {
uint32_t flags = 0;
uint32_t old_info;
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
int32_t exit_point;
const void *exit_addr;
uint8_t type;
zend_jit_addr val_addr = prop_addr;
if ((opline->op1_type & (IS_VAR|IS_TMP_VAR))
&& !delayed_fetch_this
&& !op1_avoid_refcounting) {
flags = ZEND_JIT_EXIT_FREE_OP1;
}
if ((opline->result_type & (IS_VAR|IS_TMP_VAR))
&& !(flags & ZEND_JIT_EXIT_FREE_OP1)
&& (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))
&& (ssa_op+1)->op1_use == ssa_op->result_def
&& zend_jit_may_avoid_refcounting(opline+1, res_info)) {
result_avoid_refcounting = 1;
ssa->var_info[ssa_op->result_def].avoid_refcounting = 1;
}
type = concrete_type(res_info);
if (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT) {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
} else {
val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| LOAD_ZVAL_ADDR r0, prop_addr
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack,
EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
exit_point = zend_jit_trace_get_exit_point(opline+1, flags);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (!type_loaded) {
type_loaded = 1;
| mov edx, dword [FCARG1a + prop_info->offset + 8]
}
| // ZVAL_DEREF()
| IF_NOT_TYPE dl, IS_REFERENCE, >1
| GET_Z_PTR r0, r0
| add r0, offsetof(zend_reference, val)
| GET_ZVAL_TYPE_INFO edx, val_addr
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
if (type < IS_STRING) {
|1:
if (type_loaded) {
| IF_NOT_TYPE dl, type, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE val_addr, type, &exit_addr
}
} else {
if (!type_loaded) {
type_loaded = 1;
| GET_ZVAL_TYPE_INFO edx, val_addr
}
|1:
| IF_NOT_TYPE dl, type, &exit_addr
}
| // ZVAL_COPY
| ZVAL_COPY_VALUE_V res_addr, -1, val_addr, res_info, ZREG_R0, ZREG_R1
if (type < IS_STRING) {
if (Z_REG(res_addr) != ZREG_FP ||
JIT_G(current_frame) == NULL ||
STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(res_addr))) != type) {
| SET_ZVAL_TYPE_INFO res_addr, type
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, edx
if (!result_avoid_refcounting) {
| TRY_ADDREF res_info, dh, r1
}
}
} else {
if (!zend_jit_zval_copy_deref(Dst, res_addr, prop_addr, ZREG_R2)) {
return 0;
}
}
}
if (op1_avoid_refcounting) {
SET_STACK_REG(JIT_G(current_frame)->stack,
EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE);
}
|.cold_code
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) {
|5:
| SET_EX_OPLINE opline, r0
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| EXT_CALL zend_jit_fetch_obj_w_slow, r0
} else if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| EXT_CALL zend_jit_fetch_obj_r_slow, r0
} else {
| EXT_CALL zend_jit_fetch_obj_is_slow, r0
}
| jmp >9
}
if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)- MAY_BE_OBJECT)) && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
|7:
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_UNDEF)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
if (op1_info & MAY_BE_ANY) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >1
}
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
|1:
| LOAD_ZVAL_ADDR FCARG1a, orig_op1_addr
} else if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, Z_STRVAL_P(member)
if (opline->opcode == ZEND_FETCH_OBJ_W) {
| EXT_CALL zend_jit_invalid_property_write, r0
| SET_ZVAL_TYPE_INFO res_addr, _IS_ERROR
} else {
| EXT_CALL zend_jit_invalid_property_read, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
}
| jmp >9
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
| jmp >9
}
}
if (!prop_info
&& may_be_dynamic
&& opline->opcode != ZEND_FETCH_OBJ_W) {
|8:
| mov FCARG2a, r0
| SET_EX_OPLINE opline, r0
if (opline->opcode != ZEND_FETCH_OBJ_IS) {
| EXT_CALL zend_jit_fetch_obj_r_dynamic, r0
} else {
| EXT_CALL zend_jit_fetch_obj_is_dynamic, r0
}
| jmp >9
}
|.code;
|9: // END
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if (opline->op1_type == IS_VAR
&& opline->opcode == ZEND_FETCH_OBJ_W
&& (op1_info & MAY_BE_RC1)) {
zend_jit_addr orig_op1_addr = OP1_ADDR();
| IF_NOT_ZVAL_REFCOUNTED orig_op1_addr, >1
| GET_ZVAL_PTR FCARG1a, orig_op1_addr
| GC_DELREF FCARG1a
| jnz >1
| SET_EX_OPLINE opline, r0
| EXT_CALL zend_jit_extract_helper, r0
|1:
} else if (!op1_avoid_refcounting) {
if (on_this) {
op1_info &= ~MAY_BE_RC1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE
&& prop_info
&& (opline->opcode != ZEND_FETCH_OBJ_W ||
!(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) ||
!ZEND_TYPE_IS_SET(prop_info->type))
&& (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) {
may_throw = 0;
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_incdec_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr res_addr = 0;
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool use_prop_guard = 0;
bool may_throw = 0;
uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0;
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
if (opline->result_type != IS_UNUSED) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, ZSTR_VAL(name)
| EXT_CALL zend_jit_invalid_property_incdec, r0
| jmp ->exception_handler
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
needs_slow_path = 1;
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + opline->extended_value]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >7
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| cmp aword [r0 + opline->extended_value + sizeof(void*) * 2], 0
| jnz >7
}
| mov r0, aword [r0 + opline->extended_value + sizeof(void*)]
| test r0, r0
| jl >7
if (!use_prop_guard) {
| IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7
}
| add FCARG1a, r0
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr
} else {
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7
needs_slow_path = 1;
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
may_throw = 1;
| SET_EX_OPLINE opline, r0
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
if (opline->result_type == IS_UNUSED) {
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_inc_typed_prop, r0
break;
case ZEND_PRE_DEC_OBJ:
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_dec_typed_prop, r0
break;
default:
ZEND_UNREACHABLE();
}
} else {
|.if X64
| LOAD_ZVAL_ADDR CARG3, res_addr
|.else
| sub r4, 12
| PUSH_ZVAL_ADDR res_addr, r0
|.endif
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_typed_prop, r0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_typed_prop, r0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_typed_prop, r0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_typed_prop, r0
break;
default:
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 12
|.endif
}
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
zend_jit_addr var_addr = prop_addr;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
may_throw = 1;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
if (Z_REG(prop_addr) != ZREG_FCARG1 || Z_OFFSET(prop_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2
| GET_ZVAL_PTR FCARG1a, var_addr
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >1
| lea FCARG1a, aword [FCARG1a + offsetof(zend_reference, val)]
|.cold_code
|1:
if (opline) {
| SET_EX_OPLINE opline, r0
}
if (opline->result_type == IS_UNUSED) {
| xor FCARG2a, FCARG2a
} else {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
}
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_typed_ref, r0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_typed_ref, r0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_typed_ref, r0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_typed_ref, r0
break;
default:
ZEND_UNREACHABLE();
}
| jmp >9
|.code
|2:
}
if (var_info & MAY_BE_LONG) {
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
| IF_NOT_ZVAL_TYPE var_addr, IS_LONG, >2
}
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R1, ZREG_R2
}
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
| LONG_OP_WITH_32BIT_CONST add, var_addr, Z_L(1)
} else {
| LONG_OP_WITH_32BIT_CONST sub, var_addr, Z_L(1)
}
| jo >3
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) {
if (opline->result_type != IS_UNUSED) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_LONG, ZREG_R0, ZREG_R2
}
}
|.cold_code
}
if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) {
if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
may_throw = 1;
}
if (var_info & MAY_BE_LONG) {
|2:
}
if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
}
if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) {
| ZVAL_COPY_VALUE res_addr, -1, var_addr, MAY_BE_ANY, ZREG_R0, ZREG_R2
| TRY_ADDREF MAY_BE_ANY, ah, r2
}
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_inc, r0
} else {
| EXT_CALL increment_function, r0
}
} else {
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| LOAD_ZVAL_ADDR FCARG2a, res_addr
| EXT_CALL zend_jit_pre_dec, r0
} else {
| EXT_CALL decrement_function, r0
}
}
if (var_info & MAY_BE_LONG) {
| jmp >4
}
}
if (var_info & MAY_BE_LONG) {
|3:
if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) {
|.if X64
| mov64 rax, 0x43e0000000000000
| SET_ZVAL_LVAL var_addr, rax
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, rax
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.else
| SET_ZVAL_LVAL var_addr, 0
| SET_ZVAL_W2 var_addr, 0x41e00000
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, 0
| SET_ZVAL_W2 res_addr, 0x41e00000
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.endif
} else {
|.if X64
| mov64 rax, 0xc3e0000000000000
| SET_ZVAL_LVAL var_addr, rax
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, rax
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.else
| SET_ZVAL_LVAL var_addr, 0x00200000
| SET_ZVAL_W2 var_addr, 0xc1e00000
| SET_ZVAL_TYPE_INFO var_addr, IS_DOUBLE
if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) {
| SET_ZVAL_LVAL res_addr, 0x00200000
| SET_ZVAL_W2 res_addr, 0xc1e00000
| SET_ZVAL_TYPE_INFO res_addr, IS_DOUBLE
}
|.endif
}
if (opline->result_type != IS_UNUSED
&& (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ)
&& prop_info
&& !ZEND_TYPE_IS_SET(prop_info->type)
&& (res_info & MAY_BE_GUARD)
&& (res_info & MAY_BE_LONG)) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
int32_t exit_point;
const void *exit_addr;
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0);
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info);
ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD;
| jmp &exit_addr
|.code
} else {
| jmp >4
|.code
|4:
}
}
}
if (needs_slow_path) {
may_throw = 1;
|.cold_code
|7:
| SET_EX_OPLINE opline, r0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2a, name
|.if X64
| mov CARG3, EX->run_time_cache
| add CARG3, opline->extended_value
if (opline->result_type == IS_UNUSED) {
| xor CARG4, CARG4
} else {
| LOAD_ZVAL_ADDR CARG4, res_addr
}
|.else
| sub r4, 8
if (opline->result_type == IS_UNUSED) {
| push 0
} else {
| PUSH_ZVAL_ADDR res_addr, r0
}
| mov r0, EX->run_time_cache
| add r0, opline->extended_value
| push r0
|.endif
switch (opline->opcode) {
case ZEND_PRE_INC_OBJ:
| EXT_CALL zend_jit_pre_inc_obj_helper, r0
break;
case ZEND_PRE_DEC_OBJ:
| EXT_CALL zend_jit_pre_dec_obj_helper, r0
break;
case ZEND_POST_INC_OBJ:
| EXT_CALL zend_jit_post_inc_obj_helper, r0
break;
case ZEND_POST_DEC_OBJ:
| EXT_CALL zend_jit_post_dec_obj_helper, r0
break;
default:
ZEND_UNREACHABLE();
}
|.if not(X64)
| add r4, 8
|.endif
| jmp >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_assign_obj_op(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
zend_ssa_range *val_range,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr val_addr = OP1_DATA_ADDR();
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool use_prop_guard = 0;
bool may_throw = 0;
binary_op_type binary_op = get_binary_op(opline->extended_value);
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
ZEND_ASSERT(opline->result_type == IS_UNUSED);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, ZSTR_VAL(name)
if (op1_info & MAY_BE_UNDEF) {
| EXT_CALL zend_jit_invalid_property_assign_op, r0
} else {
| EXT_CALL zend_jit_invalid_property_assign, r0
}
may_throw = 1;
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
| jmp >8
} else {
| jmp >9
}
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
use_prop_guard = (prop_type != IS_UNKNOWN
&& prop_type != IS_UNDEF
&& prop_type != IS_REFERENCE
&& (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT);
if (!prop_info) {
needs_slow_path = 1;
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + (opline+1)->extended_value]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >7
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| cmp aword [r0 + (opline+1)->extended_value + sizeof(void*) * 2], 0
| jnz >7
}
| mov r0, aword [r0 + (opline+1)->extended_value + sizeof(void*)]
| test r0, r0
| jl >7
if (!use_prop_guard) {
| IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >7
}
| add FCARG1a, r0
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr
} else {
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >7
needs_slow_path = 1;
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t info = val_info;
may_throw = 1;
if (opline) {
| SET_EX_OPLINE opline, r0
}
| IF_ZVAL_TYPE prop_addr, IS_REFERENCE, >1
|.cold_code
|1:
| GET_ZVAL_PTR FCARG1a, prop_addr
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
|.code
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
| LOAD_ADDR CARG4, binary_op
|.else
| sub r4, 8
| PUSH_ADDR binary_op, r0
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_op_to_typed_prop, r0
|.if not(X64)
| add r4, 8
|.endif
if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
info |= MAY_BE_RC1|MAY_BE_RCN;
}
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
zend_jit_addr var_addr = prop_addr;
uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN;
if (use_prop_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, prop_type, &exit_addr
var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF));
}
if (var_info & MAY_BE_REF) {
may_throw = 1;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
| LOAD_ZVAL_ADDR r0, prop_addr
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, >2
| GET_ZVAL_PTR FCARG1a, var_addr
| cmp aword [FCARG1a + offsetof(zend_reference, sources.ptr)], 0
| jnz >1
| lea r0, aword [FCARG1a + offsetof(zend_reference, val)]
|.cold_code
|1:
if (Z_MODE(val_addr) != IS_MEM_ZVAL || Z_REG(val_addr) != ZREG_FCARG2 || Z_OFFSET(val_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG2a, val_addr
}
if (opline) {
| SET_EX_OPLINE opline, r0
}
|.if X64
| LOAD_ADDR CARG3, binary_op
|.else
| sub r4, 12
| PUSH_ADDR binary_op, r0
|.endif
if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR))
&& (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| EXT_CALL zend_jit_assign_op_to_typed_ref_tmp, r0
} else {
| EXT_CALL zend_jit_assign_op_to_typed_ref, r0
}
|.if not(X64)
| add r4, 12
|.endif
| jmp >9
|.code
|2:
var_info &= ~MAY_BE_REF;
}
switch (opline->extended_value) {
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if (!zend_jit_math_helper(Dst, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info,
1 /* may overflow */, 0)) {
return 0;
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
may_throw = 1;
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if ((var_info & MAY_BE_ANY) != MAY_BE_STRING ||
(val_info & MAY_BE_ANY) != MAY_BE_STRING) {
may_throw = 1;
}
}
goto long_math;
case ZEND_SL:
case ZEND_SR:
if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
may_throw = 1;
}
if ((opline+1)->op1_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) {
may_throw = 1;
}
goto long_math;
case ZEND_MOD:
if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) ||
(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
if (opline->extended_value != ZEND_ADD ||
(var_info & MAY_BE_ANY) != MAY_BE_ARRAY ||
(val_info & MAY_BE_ANY) == MAY_BE_ARRAY) {
may_throw = 1;
}
}
if ((opline+1)->op1_type != IS_CONST ||
Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG ||
Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) {
may_throw = 1;
}
long_math:
if (!zend_jit_long_math_helper(Dst, opline, opline->extended_value,
IS_CV, opline->op1, var_addr, var_info, NULL,
(opline+1)->op1_type, (opline+1)->op1, val_addr, val_info,
val_range,
0, var_addr, var_def_info, var_info, /* may throw */ 1)) {
return 0;
}
break;
case ZEND_CONCAT:
may_throw = 1;
if (!zend_jit_concat_helper(Dst, opline, IS_CV, opline->op1, var_addr, var_info, (opline+1)->op1_type, (opline+1)->op1, val_addr, val_info, var_addr,
0)) {
return 0;
}
break;
default:
ZEND_UNREACHABLE();
}
}
if (needs_slow_path) {
may_throw = 1;
|.cold_code
|7:
| SET_EX_OPLINE opline, r0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2a, name
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
| mov CARG4, EX->run_time_cache
| add CARG4, (opline+1)->extended_value
|.if X64WIN
| LOAD_ADDR r0, binary_op
| mov aword A5, r0
|.else
| LOAD_ADDR CARG5, binary_op
|.endif
|.else
| sub r4, 4
| PUSH_ADDR binary_op, r0
| mov r0, EX->run_time_cache
| add r0, (opline+1)->extended_value
| push r0
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_obj_op_helper, r0
|.if not(X64)
| add r4, 4
|.endif
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
|8:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline
| jmp >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) {
may_throw = 1;
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_assign_obj(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
uint32_t op1_info,
zend_jit_addr op1_addr,
uint32_t val_info,
bool op1_indirect,
zend_class_entry *ce,
bool ce_is_instanceof,
bool on_this,
bool delayed_fetch_this,
zend_class_entry *trace_ce,
uint8_t prop_type,
int may_throw)
{
zval *member;
zend_string *name;
zend_property_info *prop_info;
zend_jit_addr val_addr = OP1_DATA_ADDR();
zend_jit_addr res_addr = 0;
zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This));
zend_jit_addr prop_addr;
bool needs_slow_path = 0;
bool needs_val_dtor = 0;
if (RETURN_VALUE_USED(opline)) {
res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
}
ZEND_ASSERT(opline->op2_type == IS_CONST);
ZEND_ASSERT(op1_info & MAY_BE_OBJECT);
member = RT_CONSTANT(opline, opline->op2);
ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0');
name = Z_STR_P(member);
prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename);
if (on_this) {
| GET_ZVAL_PTR FCARG1a, this_addr
} else {
if (opline->op1_type == IS_VAR
&& (op1_info & MAY_BE_INDIRECT)
&& Z_REG(op1_addr) == ZREG_FP) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
| IF_NOT_Z_TYPE FCARG1a, IS_INDIRECT, >1
| GET_Z_PTR FCARG1a, FCARG1a
|1:
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & MAY_BE_REF) {
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| ZVAL_DEREF FCARG1a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, &exit_addr
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_OBJECT, >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
if (Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
}
| LOAD_ADDR FCARG2a, ZSTR_VAL(name)
| EXT_CALL zend_jit_invalid_property_assign, r0
if (RETURN_VALUE_USED(opline)) {
| SET_ZVAL_TYPE_INFO res_addr, IS_NULL
}
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
needs_val_dtor = 1;
| jmp >7
} else {
| jmp >9
}
|.code
}
}
| GET_ZVAL_PTR FCARG1a, op1_addr
}
if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) {
prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename);
if (prop_info) {
ce = trace_ce;
ce_is_instanceof = 0;
if (!(op1_info & MAY_BE_CLASS_GUARD)) {
if (on_this && JIT_G(current_frame)
&& TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) {
ZEND_ASSERT(JIT_G(current_frame)->ce == ce);
} else if (zend_jit_class_guard(Dst, opline, ce)) {
if (on_this && JIT_G(current_frame)) {
JIT_G(current_frame)->ce = ce;
TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame));
}
} else {
return 0;
}
if (ssa->var_info && ssa_op->op1_use >= 0) {
ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_use].ce = ce;
ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof;
}
if (ssa->var_info && ssa_op->op1_def >= 0) {
ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD;
ssa->var_info[ssa_op->op1_def].ce = ce;
ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof;
}
}
}
}
if (!prop_info) {
needs_slow_path = 1;
| mov r0, EX->run_time_cache
| mov r2, aword [r0 + opline->extended_value]
| cmp r2, aword [FCARG1a + offsetof(zend_object, ce)]
| jne >5
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| mov FCARG2a, aword [r0 + opline->extended_value + sizeof(void*) * 2]
}
| mov r0, aword [r0 + opline->extended_value + sizeof(void*)]
| test r0, r0
| jl >5
| IF_TYPE byte [FCARG1a + r0 + 8], IS_UNDEF, >5
| add FCARG1a, r0
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) {
| test FCARG2a, FCARG2a
| jnz >1
|.cold_code
|1:
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
| SET_EX_OPLINE opline, r0
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR CARG4, res_addr
} else {
| xor CARG4, CARG4
}
|.else
| sub r4, 8
if (RETURN_VALUE_USED(opline)) {
| PUSH_ZVAL_ADDR res_addr, r0
} else {
| push 0
}
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_to_typed_prop, r0
|.if not(X64)
| add r4, 8
|.endif
if ((opline+1)->op1_type == IS_CONST) {
| // TODO: ???
| // if (Z_TYPE_P(value) == orig_type) {
| // CACHE_PTR_EX(cache_slot + 2, NULL);
}
if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR))
&& (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) {
| jmp >7
} else {
| jmp >9
}
|.code
}
} else {
prop_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, prop_info->offset);
if (!ce || ce_is_instanceof || !(ce->ce_flags & ZEND_ACC_IMMUTABLE) || ce->__get || ce->__set || (prop_info->flags & ZEND_ACC_READONLY)) {
// Undefined property with magic __get()/__set()
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, &exit_addr
} else {
| IF_TYPE byte [FCARG1a + prop_info->offset + 8], IS_UNDEF, >5
needs_slow_path = 1;
}
}
if (ZEND_TYPE_IS_SET(prop_info->type)) {
uint32_t info = val_info;
| // value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC);
| SET_EX_OPLINE opline, r0
if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) {
| LOAD_ADDR FCARG2a, prop_info
} else {
int prop_info_offset =
(((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*));
| mov r0, aword [FCARG1a + offsetof(zend_object, ce)]
| mov r0, aword [r0 + offsetof(zend_class_entry, properties_info_table)]
| mov FCARG2a, aword[r0 + prop_info_offset]
}
| LOAD_ZVAL_ADDR FCARG1a, prop_addr
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
if (RETURN_VALUE_USED(opline)) {
| LOAD_ZVAL_ADDR CARG4, res_addr
} else {
| xor CARG4, CARG4
}
|.else
| sub r4, 8
if (RETURN_VALUE_USED(opline)) {
| PUSH_ZVAL_ADDR res_addr, r0
} else {
| push 0
}
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_to_typed_prop, r0
|.if not(X64)
| add r4, 8
|.endif
if (info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
info |= MAY_BE_RC1|MAY_BE_RCN;
}
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, info, 0, NULL
}
}
if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) {
// value = zend_assign_to_variable(property_val, value, OP_DATA_TYPE, EX_USES_STRICT_TYPES());
if (opline->result_type == IS_UNUSED) {
if (!zend_jit_assign_to_variable_call(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
} else {
if (!zend_jit_assign_to_variable(Dst, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) {
return 0;
}
}
}
if (needs_slow_path) {
|.cold_code
|5:
| SET_EX_OPLINE opline, r0
| // value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value));
| LOAD_ADDR FCARG2a, name
|.if X64
| LOAD_ZVAL_ADDR CARG3, val_addr
| mov CARG4, EX->run_time_cache
| add CARG4, opline->extended_value
if (RETURN_VALUE_USED(opline)) {
|.if X64WIN
| LOAD_ZVAL_ADDR r0, res_addr
| mov aword A5, r0
|.else
| LOAD_ZVAL_ADDR CARG5, res_addr
|.endif
} else {
|.if X64WIN
| mov aword A5, 0
|.else
| xor CARG5, CARG5
|.endif
}
|.else
| sub r4, 4
if (RETURN_VALUE_USED(opline)) {
| PUSH_ZVAL_ADDR res_addr, r0
} else {
| push 0
}
| mov r0, EX->run_time_cache
| add r0, opline->extended_value
| push r0
| PUSH_ZVAL_ADDR val_addr, r0
|.endif
| EXT_CALL zend_jit_assign_obj_helper, r0
|.if not(X64)
| add r4, 4
|.endif
if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1|MAY_BE_RCN;
}
|7:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline
| jmp >9
|.code
} else if (needs_val_dtor) {
|.cold_code
|7:
| // FREE_OP_DATA();
| FREE_OP (opline+1)->op1_type, (opline+1)->op1, val_info, 0, opline
| jmp >9
|.code
}
|9:
if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) {
| FREE_OP opline->op1_type, opline->op1, op1_info, 1, opline
}
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_free(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, int may_throw)
{
zend_jit_addr op1_addr = OP1_ADDR();
if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {
if (may_throw) {
| SET_EX_OPLINE opline, r0
}
if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) {
if (op1_info & MAY_BE_ARRAY) {
| IF_ZVAL_TYPE op1_addr, IS_ARRAY, >7
}
| mov FCARG1d, dword [FP + opline->op1.var + offsetof(zval, u2.fe_iter_idx)]
| cmp FCARG1d, -1
| je >7
| EXT_CALL zend_hash_iterator_del, r0
|7:
}
| ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline
if (may_throw) {
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
}
return 1;
}
static int zend_jit_echo(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
if (len > 0) {
const char *str = Z_STRVAL_P(zv);
| SET_EX_OPLINE opline, r0
|.if X64
| LOAD_ADDR CARG1, str
| LOAD_ADDR CARG2, len
| EXT_CALL zend_write, r0
|.else
| mov aword A2, len
| mov aword A1, str
| EXT_CALL zend_write, r0
|.endif
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
} else {
zend_jit_addr op1_addr = OP1_ADDR();
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
| SET_EX_OPLINE opline, r0
| GET_ZVAL_PTR r0, op1_addr
|.if X64
| lea CARG1, aword [r0 + offsetof(zend_string, val)]
| mov CARG2, aword [r0 + offsetof(zend_string, len)]
| EXT_CALL zend_write, r0
|.else
| add r0, offsetof(zend_string, val)
| mov aword A1, r0
| mov r0, aword [r0 + (offsetof(zend_string, len)-offsetof(zend_string, val))]
| mov aword A2, r0
| EXT_CALL zend_write, r0
|.endif
if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) {
| ZVAL_PTR_DTOR op1_addr, op1_info, 0, 0, opline
}
if (!zend_jit_check_exception(Dst)) {
return 0;
}
}
return 1;
}
static int zend_jit_strlen(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
size_t len;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
len = Z_STRLEN_P(zv);
| SET_ZVAL_LVAL res_addr, len
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr
| mov Ra(Z_REG(res_addr)), aword [Ra(Z_REG(res_addr))+offsetof(zend_string, len)]
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
| GET_ZVAL_PTR r0, op1_addr
| mov r0, aword [r0 + offsetof(zend_string, len)]
| SET_ZVAL_LVAL res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
return 1;
}
static int zend_jit_count(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw)
{
if (opline->op1_type == IS_CONST) {
zval *zv;
zend_long count;
zv = RT_CONSTANT(opline, opline->op1);
ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY);
count = zend_hash_num_elements(Z_ARRVAL_P(zv));
| SET_ZVAL_LVAL res_addr, count
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY);
// Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+.
if (Z_MODE(res_addr) == IS_REG) {
| GET_ZVAL_PTR Ra(Z_REG(res_addr)), op1_addr
// Sign-extend the 32-bit value to a potentially 64-bit zend_long
| mov Rd(Z_REG(res_addr)), dword [Ra(Z_REG(res_addr))+offsetof(HashTable, nNumOfElements)]
if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, MAY_BE_LONG)) {
return 0;
}
} else {
| GET_ZVAL_PTR r0, op1_addr
// Sign-extend the 32-bit value to a potentially 64-bit zend_long
| mov eax, dword [r0 + offsetof(HashTable, nNumOfElements)]
| SET_ZVAL_LVAL res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
| FREE_OP opline->op1_type, opline->op1, op1_info, 0, opline
}
if (may_throw) {
return zend_jit_check_exception(Dst);
}
return 1;
}
static int zend_jit_load_this(dasm_State **Dst, uint32_t var)
{
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var);
| mov FCARG1a, aword EX->This.value.ptr
| SET_ZVAL_PTR var_addr, FCARG1a
| SET_ZVAL_TYPE_INFO var_addr, IS_OBJECT_EX
| GC_ADDREF FCARG1a
return 1;
}
static int zend_jit_fetch_this(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, bool check_only)
{
if (!op_array->scope ||
(op_array->fn_flags & ZEND_ACC_STATIC) ||
((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_IMMUTABLE)) == ZEND_ACC_CLOSURE)) {
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (!JIT_G(current_frame) ||
!TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| cmp byte EX->This.u1.v.type, IS_OBJECT
| jne &exit_addr
if (JIT_G(current_frame)) {
TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame));
}
}
} else {
| cmp byte EX->This.u1.v.type, IS_OBJECT
| jne >1
|.cold_code
|1:
| SET_EX_OPLINE opline, r0
| jmp ->invalid_this
|.code
}
}
if (!check_only) {
if (!zend_jit_load_this(Dst, opline->result.var)) {
return 0;
}
}
return 1;
}
static int zend_jit_hash_jmp(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, HashTable *jumptable, int default_b, const void *default_label, const zend_op *next_opline, zend_jit_trace_info *trace_info)
{
uint32_t count;
Bucket *p;
const zend_op *target;
int b;
int32_t exit_point;
const void *exit_addr;
| test r0, r0
if (default_label) {
| jz &default_label
} else if (next_opline) {
| jz >3
} else {
| jz =>default_b
}
| LOAD_ADDR FCARG1a, jumptable
| sub r0, aword [FCARG1a + offsetof(HashTable, arData)]
if (HT_IS_PACKED(jumptable)) {
| mov FCARG1a, (sizeof(zval) / sizeof(void*))
} else {
| mov FCARG1a, (sizeof(Bucket) / sizeof(void*))
}
|.if X64
| cqo
|.else
| cdq
|.endif
| idiv FCARG1a
|.if X64
if (!IS_32BIT(dasm_end)) {
| lea FCARG1a, aword [>4]
| jmp aword [FCARG1a + r0]
} else {
| jmp aword [r0 + >4]
}
|.else
| jmp aword [r0 + >4]
|.endif
|.jmp_table
|.align aword
|4:
if (trace_info) {
trace_info->jmp_table_size += zend_hash_num_elements(jumptable);
}
count = jumptable->nNumUsed;
p = jumptable->arData;
do {
if (Z_TYPE(p->val) == IS_UNDEF) {
if (default_label) {
| .aword &default_label
} else if (next_opline) {
| .aword >3
} else {
| .aword =>default_b
}
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL(p->val));
if (!next_opline) {
b = ssa->cfg.map[target - op_array->opcodes];
| .aword =>b
} else if (next_opline == target) {
| .aword >3
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| .aword &exit_addr
}
}
if (HT_IS_PACKED(jumptable)) {
p = (Bucket*)(((zval*)p)+1);
} else {
p++;
}
count--;
} while (count);
|.code
return 1;
}
static int zend_jit_switch(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info)
{
HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
const zend_op *next_opline = NULL;
if (trace) {
ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END);
ZEND_ASSERT(trace->opline != NULL);
next_opline = trace->opline;
}
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
zval *jump_zv = NULL;
int b;
if (opline->opcode == ZEND_SWITCH_LONG) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else if (opline->opcode == ZEND_MATCH) {
if (Z_TYPE_P(zv) == IS_LONG) {
jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv));
} else if (Z_TYPE_P(zv) == IS_STRING) {
jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv));
}
} else {
ZEND_UNREACHABLE();
}
if (next_opline) {
const zend_op *target;
if (jump_zv != NULL) {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv));
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
}
ZEND_ASSERT(target == next_opline);
} else {
if (jump_zv != NULL) {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes];
} else {
b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes];
}
| jmp =>b
}
} else {
zend_ssa_op *ssa_op = &ssa->ops[opline - op_array->opcodes];
uint32_t op1_info = OP1_INFO();
zend_jit_addr op1_addr = OP1_ADDR();
const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value);
const zend_op *target;
int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes];
int b;
int32_t exit_point;
const void *fallback_label = NULL;
const void *default_label = NULL;
const void *exit_addr;
if (next_opline) {
if (next_opline != opline + 1) {
exit_point = zend_jit_trace_get_exit_point(opline + 1, 0);
fallback_label = zend_jit_trace_get_exit_addr(exit_point);
if (!fallback_label) {
return 0;
}
}
if (next_opline != default_opline) {
exit_point = zend_jit_trace_get_exit_point(default_opline, 0);
default_label = zend_jit_trace_get_exit_addr(exit_point);
if (!default_label) {
return 0;
}
}
}
if (opline->opcode == ZEND_SWITCH_LONG) {
if (op1_info & MAY_BE_LONG) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >1
| GET_ZVAL_LVAL ZREG_FCARG2, op1_addr
|.cold_code
|1:
| // ZVAL_DEREF(op)
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
}
| GET_ZVAL_PTR FCARG2a, op1_addr
if (fallback_label) {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, &fallback_label
} else {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_LONG, >3
}
| mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.lval)]
| jmp >2
|.code
|2:
} else {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
}
}
| GET_ZVAL_LVAL ZREG_FCARG2, op1_addr
}
if (HT_IS_PACKED(jumptable)) {
uint32_t count = jumptable->nNumUsed;
zval *zv = jumptable->arPacked;
| cmp FCARG2a, jumptable->nNumUsed
if (default_label) {
| jae &default_label
} else if (next_opline) {
| jae >3
} else {
| jae =>default_b
}
|.if X64
if (!IS_32BIT(dasm_end)) {
| lea r0, aword [>4]
| jmp aword [r0 + FCARG2a * 8]
} else {
| jmp aword [FCARG2a * 8 + >4]
}
|.else
| jmp aword [FCARG2a * 4 + >4]
|.endif
|.jmp_table
|.align aword
|4:
if (trace_info) {
trace_info->jmp_table_size += count;
}
do {
if (Z_TYPE_P(zv) == IS_UNDEF) {
if (default_label) {
| .aword &default_label
} else if (next_opline) {
| .aword >3
} else {
| .aword =>default_b
}
} else {
target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv));
if (!next_opline) {
b = ssa->cfg.map[target - op_array->opcodes];
| .aword =>b
} else if (next_opline == target) {
| .aword >3
} else {
exit_point = zend_jit_trace_get_exit_point(target, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| .aword &exit_addr
}
}
zv++;
count--;
} while (count);
|.code
|3:
} else {
| LOAD_ADDR FCARG1a, jumptable
| EXT_CALL zend_hash_index_find, r0
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
|3:
}
}
} else if (opline->opcode == ZEND_SWITCH_STRING) {
if (op1_info & MAY_BE_STRING) {
if (op1_info & MAY_BE_REF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >1
| GET_ZVAL_PTR FCARG2a, op1_addr
|.cold_code
|1:
| // ZVAL_DEREF(op)
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_REFERENCE, >3
}
| GET_ZVAL_PTR FCARG2a, op1_addr
if (fallback_label) {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, &fallback_label
} else {
| IF_NOT_Z_TYPE FCARG2a + offsetof(zend_reference, val), IS_STRING, >3
}
| mov FCARG2a, aword [FCARG2a + offsetof(zend_reference, val.value.ptr)]
| jmp >2
|.code
|2:
} else {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) {
if (fallback_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &fallback_label
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3
}
}
| GET_ZVAL_PTR FCARG2a, op1_addr
}
| LOAD_ADDR FCARG1a, jumptable
| EXT_CALL zend_hash_find, r0
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
|3:
}
} else if (opline->opcode == ZEND_MATCH) {
if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) {
if (op1_info & MAY_BE_REF) {
| LOAD_ZVAL_ADDR FCARG2a, op1_addr
| ZVAL_DEREF FCARG2a, op1_info
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
}
| LOAD_ADDR FCARG1a, jumptable
if (op1_info & MAY_BE_LONG) {
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) {
if (op1_info & MAY_BE_STRING) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >5
} else if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >6
} else if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, &default_label
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_LONG, =>default_b
}
}
| GET_ZVAL_LVAL ZREG_FCARG2, op1_addr
| EXT_CALL zend_hash_index_find, r0
if (op1_info & MAY_BE_STRING) {
| jmp >2
}
}
if (op1_info & MAY_BE_STRING) {
|5:
if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) {
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >6
} else if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, &default_label
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_STRING, =>default_b
}
}
| GET_ZVAL_PTR FCARG2a, op1_addr
| EXT_CALL zend_hash_find, r0
}
|2:
if (!zend_jit_hash_jmp(Dst, opline, op_array, ssa, jumptable, default_b, default_label, next_opline, trace_info)) {
return 0;
}
}
if (op1_info & MAY_BE_UNDEF) {
|6:
if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) {
if (default_label) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, &default_label
} else if (next_opline) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >3
} else {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, =>default_b
}
}
| // zend_error(E_WARNING, "Undefined variable $%s", ZSTR_VAL(CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))));
| SET_EX_OPLINE opline, r0
| mov FCARG1d, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, r0
if (!zend_jit_check_exception_undef_result(Dst, opline)) {
return 0;
}
}
if (default_label) {
| jmp &default_label
} else if (next_opline) {
| jmp >3
} else {
| jmp =>default_b
}
|3:
} else {
ZEND_UNREACHABLE();
}
}
return 1;
}
static bool zend_jit_verify_return_type(dasm_State **Dst, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info)
{
zend_arg_info *arg_info = &op_array->arg_info[-1];
ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type));
zend_jit_addr op1_addr = OP1_ADDR();
bool needs_slow_check = 1;
bool slow_check_in_cold = 1;
uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY;
if (type_mask == 0) {
slow_check_in_cold = 0;
} else {
if (((op1_info & MAY_BE_ANY) & type_mask) == 0) {
slow_check_in_cold = 0;
} else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) {
needs_slow_check = 0;
} else if (is_power_of_two(type_mask)) {
uint32_t type_code = concrete_type(type_mask);
| IF_NOT_ZVAL_TYPE op1_addr, type_code, >6
} else {
| mov edx, 1
| GET_ZVAL_TYPE cl, op1_addr
| shl edx, cl
| test edx, type_mask
| je >6
}
}
if (needs_slow_check) {
if (slow_check_in_cold) {
|.cold_code
|6:
}
| SET_EX_OPLINE opline, r1
if (op1_info & MAY_BE_UNDEF) {
| IF_NOT_ZVAL_TYPE op1_addr, IS_UNDEF, >7
| mov FCARG1a, opline->op1.var
| EXT_CALL zend_jit_undefined_op_helper, FCARG2a
| test r0, r0
| jz ->exception_handler
| LOAD_ADDR_ZTS FCARG1a, executor_globals, uninitialized_zval
| jmp >8
}
|7:
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
|8:
| mov FCARG2a, EX->func
|.if X64
| LOAD_ADDR CARG3, (ptrdiff_t)arg_info
| mov r0, EX->run_time_cache
| lea CARG4, aword [r0+opline->op2.num]
| EXT_CALL zend_jit_verify_return_slow, r0
|.else
| sub r4, 8
| mov r0, EX->run_time_cache
| add r0, opline->op2.num
| push r0
| push (ptrdiff_t)arg_info
| EXT_CALL zend_jit_verify_return_slow, r0
| add r4, 8
|.endif
if (!zend_jit_check_exception(Dst)) {
return 0;
}
if (slow_check_in_cold) {
| jmp >9
|.code
}
}
|9:
return 1;
}
static int zend_jit_isset_isempty_cv(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
// TODO: support for empty() ???
ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY));
if (op1_info & MAY_BE_REF) {
if (Z_MODE(op1_addr) != IS_MEM_ZVAL || Z_REG(op1_addr) != ZREG_FCARG1 || Z_OFFSET(op1_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, op1_addr
op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
}
| ZVAL_DEREF FCARG1a, op1_info
|1:
}
if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ);
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jmp =>target_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_TRUE
}
} else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) {
if (exit_addr) {
ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ);
} else if (smart_branch_opcode) {
if (smart_branch_opcode != ZEND_JMPNZ) {
| jmp =>target_label
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, IS_FALSE
}
} else {
ZEND_ASSERT(Z_MODE(op1_addr) == IS_MEM_ZVAL);
| cmp byte [Ra(Z_REG(op1_addr))+Z_OFFSET(op1_addr)+offsetof(zval, u1.v.type)], IS_NULL
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPNZ) {
| jg &exit_addr
} else {
| jle &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jle =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jg =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
| setg al
| movzx eax, al
| lea eax, [eax + IS_FALSE]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
}
return 1;
}
static int zend_jit_fe_reset(dasm_State **Dst, const zend_op *opline, uint32_t op1_info)
{
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
if (opline->op1_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op1);
| ZVAL_COPY_CONST res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, ZREG_R0
if (Z_REFCOUNTED_P(zv)) {
| ADDREF_CONST zv, r0
}
} else {
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
| // ZVAL_COPY(res, value);
| ZVAL_COPY_VALUE res_addr, -1, op1_addr, op1_info, ZREG_R0, ZREG_FCARG1
if (opline->op1_type == IS_CV) {
| TRY_ADDREF op1_info, ah, FCARG1a
}
}
| // Z_FE_POS_P(res) = 0;
| mov dword [FP + opline->result.var + offsetof(zval, u2.fe_pos)], 0
return 1;
}
static int zend_jit_fe_fetch(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr)
{
zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var);
if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) {
/* empty array */
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
| jmp &exit_addr
}
} else {
| jmp =>target_label
}
return 1;
}
| // array = EX_VAR(opline->op1.var);
| // fe_ht = Z_ARRVAL_P(array);
| GET_ZVAL_PTR FCARG1a, op1_addr
if (op1_info & MAY_BE_PACKED_GUARD) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
if (op1_info & MAY_BE_ARRAY_PACKED) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jz &exit_addr
} else {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jnz &exit_addr
}
}
| // pos = Z_FE_POS_P(array);
| mov eax, dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)]
if (MAY_BE_HASH(op1_info)) {
if (MAY_BE_PACKED(op1_info)) {
| test dword [FCARG1a + offsetof(zend_array, u.flags)], HASH_FLAG_PACKED
| jnz >2
}
| // p = fe_ht->arData + pos;
|.if X64
|| ZEND_ASSERT(sizeof(Bucket) == 32);
| mov FCARG2d, eax
| shl FCARG2a, 5
|.else
| imul FCARG2a, r0, sizeof(Bucket)
|.endif
| add FCARG2a, aword [FCARG1a + offsetof(zend_array, arData)]
|1:
| // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
| cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax
| // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
| // ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
| jbe &exit_addr
} else {
| jbe >3
}
} else {
| jbe =>target_label
}
| // pos++;
| add eax, 1
| // value_type = Z_TYPE_INFO_P(value);
| // if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
| IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >3
} else {
| IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr
}
| // p++;
| add FCARG2a, sizeof(Bucket)
| jmp <1
if (MAY_BE_PACKED(op1_info)) {
|2:
}
}
if (MAY_BE_PACKED(op1_info)) {
| // p = fe_ht->arPacked + pos;
|| ZEND_ASSERT(sizeof(zval) == 16);
| mov FCARG2d, eax
| shl FCARG2a, 4
| add FCARG2a, aword [FCARG1a + offsetof(zend_array, arPacked)]
|1:
| // if (UNEXPECTED(pos >= fe_ht->nNumUsed)) {
| cmp dword [FCARG1a + offsetof(zend_array, nNumUsed)], eax
| // ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value);
| // ZEND_VM_CONTINUE();
if (exit_addr) {
if (exit_opcode == ZEND_JMP) {
| jbe &exit_addr
} else {
| jbe >4
}
} else {
| jbe =>target_label
}
| // pos++;
| add eax, 1
| // value_type = Z_TYPE_INFO_P(value);
| // if (EXPECTED(value_type != IS_UNDEF)) {
if (!exit_addr || exit_opcode == ZEND_JMP) {
| IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, >4
} else {
| IF_NOT_Z_TYPE FCARG2a, IS_UNDEF, &exit_addr
}
| // p++;
| add FCARG2a, sizeof(zval)
| jmp <1
}
if (!exit_addr || exit_opcode == ZEND_JMP) {
zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG2, 0);
zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var);
uint32_t val_info;
if (RETURN_VALUE_USED(opline)) {
zend_jit_addr res_addr = RES_ADDR();
if (MAY_BE_HASH(op1_info)) {
|3:
| // Z_FE_POS_P(array) = pos + 1;
| mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax
if ((op1_info & MAY_BE_ARRAY_KEY_LONG)
&& (op1_info & MAY_BE_ARRAY_KEY_STRING)) {
| // if (!p->key) {
| cmp aword [FCARG2a + offsetof(Bucket, key)], 0
| jz >2
}
if (op1_info & MAY_BE_ARRAY_KEY_STRING) {
| // ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key);
| mov r0, aword [FCARG2a + offsetof(Bucket, key)]
| SET_ZVAL_PTR res_addr, r0
| test dword [r0 + offsetof(zend_refcounted, gc.u.type_info)], IS_STR_INTERNED
| jz >1
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING
| jmp >3
|1:
| GC_ADDREF r0
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX
if ((op1_info & MAY_BE_ARRAY_KEY_LONG) || MAY_BE_PACKED(op1_info)) {
| jmp >3
|2:
}
}
if (op1_info & MAY_BE_ARRAY_KEY_LONG) {
| // ZVAL_LONG(EX_VAR(opline->result.var), p->h);
| mov r0, aword [FCARG2a + offsetof(Bucket, h)]
| SET_ZVAL_LVAL res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
if (MAY_BE_PACKED(op1_info)) {
| jmp >3
}
}
}
if (MAY_BE_PACKED(op1_info)) {
|4:
| // Z_FE_POS_P(array) = pos + 1;
| mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax
| sub r0, 1
| SET_ZVAL_LVAL res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_LONG
}
|3:
} else {
|3:
|4:
| // Z_FE_POS_P(array) = pos + 1;
| mov dword [FP + opline->op1.var + offsetof(zval, u2.fe_pos)], eax
}
val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT);
if (val_info & MAY_BE_ARRAY) {
val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
}
if (op1_info & MAY_BE_ARRAY_OF_REF) {
val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY |
MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF;
} else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) {
val_info |= MAY_BE_RC1 | MAY_BE_RCN;
}
if (opline->op2_type == IS_CV) {
| // zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES());
if (!zend_jit_assign_to_variable(Dst, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 1)) {
return 0;
}
} else {
| // ZVAL_COPY(res, value);
| ZVAL_COPY_VALUE var_addr, -1, val_addr, val_info, ZREG_R0, ZREG_FCARG1
| TRY_ADDREF val_info, ah, FCARG1a
}
} else {
|3:
|4:
}
return 1;
}
static int zend_jit_fetch_constant(dasm_State **Dst,
const zend_op *opline,
const zend_op_array *op_array,
zend_ssa *ssa,
const zend_ssa_op *ssa_op,
zend_jit_addr res_addr)
{
zval *zv = RT_CONSTANT(opline, opline->op2) + 1;
zend_jit_addr const_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0);
uint32_t res_info = RES_INFO();
| // c = CACHED_PTR(opline->extended_value);
| mov FCARG1a, EX->run_time_cache
| mov r0, aword [FCARG1a + opline->extended_value]
| // if (c != NULL)
| test r0, r0
| jz >9
if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) {
| // if (!IS_SPECIAL_CACHE_VAL(c))
| test r0, CACHE_SPECIAL
| jnz >9
}
|8:
if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) {
zend_jit_trace_stack *stack = JIT_G(current_frame)->stack;
uint32_t old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var));
int32_t exit_point;
const void *exit_addr = NULL;
SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1);
SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_ZVAL_COPY_GPR0);
exit_point = zend_jit_trace_get_exit_point(opline+1, 0);
SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
res_info &= ~MAY_BE_GUARD;
ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD;
uint8_t type = concrete_type(res_info);
if (type < IS_STRING) {
| IF_NOT_ZVAL_TYPE const_addr, type, &exit_addr
} else {
| GET_ZVAL_TYPE_INFO edx, const_addr
| IF_NOT_TYPE dl, type, &exit_addr
}
| ZVAL_COPY_VALUE_V res_addr, -1, const_addr, res_info, ZREG_R0, ZREG_R1
if (type < IS_STRING) {
if (Z_MODE(res_addr) == IS_MEM_ZVAL) {
| SET_ZVAL_TYPE_INFO res_addr, type
} else if (!zend_jit_store_var_if_necessary(Dst, opline->result.var, res_addr, res_info)) {
return 0;
}
} else {
| SET_ZVAL_TYPE_INFO res_addr, edx
| TRY_ADDREF res_info, dh, r1
}
} else {
| // ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup)
| ZVAL_COPY_VALUE res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, ZREG_R0, ZREG_R1
| TRY_ADDREF MAY_BE_ANY, ah, r1
}
|.cold_code
|9:
| // SAVE_OPLINE();
| SET_EX_OPLINE opline, r0
| // zend_quick_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num OPLINE_CC EXECUTE_DATA_CC);
| LOAD_ADDR FCARG1a, zv
| mov FCARG2a, opline->op1.num
| EXT_CALL zend_jit_get_constant, r0
| // ZEND_VM_NEXT_OPCODE_CHECK_EXCEPTION();
| test r0, r0
| jnz <8
| jmp ->exception_handler
|.code
return 1;
}
static int zend_jit_in_array(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr)
{
HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2));
zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var);
ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR);
ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING);
| // result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST);
| LOAD_ADDR FCARG1a, ht
if (opline->op1_type != IS_CONST) {
| GET_ZVAL_PTR FCARG2a, op1_addr
| EXT_CALL zend_hash_find, r0
} else {
zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1));
| LOAD_ADDR FCARG2a, str
| EXT_CALL zend_hash_find_known_hash, r0
}
| test r0, r0
if (exit_addr) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jz &exit_addr
} else {
| jnz &exit_addr
}
} else if (smart_branch_opcode) {
if (smart_branch_opcode == ZEND_JMPZ) {
| jz =>target_label
} else if (smart_branch_opcode == ZEND_JMPNZ) {
| jnz =>target_label
} else {
ZEND_UNREACHABLE();
}
} else {
| setnz al
| movzx eax, al
| lea eax, [eax + IS_FALSE]
| SET_ZVAL_TYPE_INFO res_addr, eax
}
return 1;
}
static int zend_jit_rope(dasm_State **Dst, const zend_op *opline, uint32_t op2_info)
{
uint32_t offset;
offset = (opline->opcode == ZEND_ROPE_INIT) ?
opline->result.var :
opline->op1.var + opline->extended_value * sizeof(zend_string*);
if (opline->op2_type == IS_CONST) {
zval *zv = RT_CONSTANT(opline, opline->op2);
zend_string *str;
ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING);
str = Z_STR_P(zv);
| ADDR_STORE aword [FP + offset], str, r0
} else {
zend_jit_addr op2_addr = OP2_ADDR();
ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING);
| GET_ZVAL_PTR r1, op2_addr
| mov aword [FP + offset], r1
if (opline->op2_type == IS_CV) {
| GET_ZVAL_TYPE_INFO eax, op2_addr
| TRY_ADDREF op2_info, ah, r1
}
}
if (opline->opcode == ZEND_ROPE_END) {
zend_jit_addr res_addr = RES_ADDR();
| lea FCARG1a, [FP + opline->op1.var]
| mov FCARG2d, opline->extended_value
| EXT_CALL zend_jit_rope_end, r0
| SET_ZVAL_PTR res_addr, r0
| SET_ZVAL_TYPE_INFO res_addr, IS_STRING_EX
}
return 1;
}
static bool zend_jit_noref_guard(dasm_State **Dst, const zend_op *opline, zend_jit_addr var_addr)
{
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr
return 1;
}
static bool zend_jit_fetch_reference(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
const void *exit_addr = NULL;
if (add_ref_guard || add_type_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
}
if (add_ref_guard) {
| IF_NOT_ZVAL_TYPE var_addr, IS_REFERENCE, &exit_addr
}
if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) {
/* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */
if (Z_REG(var_addr) != ZREG_FCARG1 || Z_OFFSET(var_addr) != 0) {
| LOAD_ZVAL_ADDR FCARG1a, var_addr
}
| EXT_CALL zend_jit_unref_helper, r0
} else {
| GET_ZVAL_PTR FCARG1a, var_addr
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, offsetof(zend_reference, val));
*var_addr_ptr = var_addr;
}
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (add_type_guard
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
| IF_NOT_ZVAL_TYPE var_addr, var_type, &exit_addr
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
} else {
var_info &= ~MAY_BE_REF;
*var_info_ptr = var_info;
}
*var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */
return 1;
}
static bool zend_jit_fetch_indirect_var(dasm_State **Dst, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard)
{
zend_jit_addr var_addr = *var_addr_ptr;
uint32_t var_info = *var_info_ptr;
int32_t exit_point;
const void *exit_addr;
if (add_indirect_guard) {
int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0);
const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_ZVAL_TYPE var_addr, IS_INDIRECT, &exit_addr
| GET_ZVAL_PTR FCARG1a, var_addr
} else {
/* May be already loaded into FCARG1a or RAX by previous FETCH_OBJ_W/DIM_W */
if (opline->op1_type != IS_VAR ||
(opline-1)->result_type != IS_VAR ||
(opline-1)->result.var != opline->op1.var ||
(opline-1)->op1_type == IS_VAR ||
(opline-1)->op2_type == IS_VAR ||
(opline-1)->op2_type == IS_TMP_VAR) {
| GET_ZVAL_PTR FCARG1a, var_addr
} else if ((opline-1)->opcode == ZEND_FETCH_DIM_W || (opline-1)->opcode == ZEND_FETCH_DIM_RW) {
| mov FCARG1a, r0
}
}
*var_info_ptr &= ~MAY_BE_INDIRECT;
var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FCARG1, 0);
*var_addr_ptr = var_addr;
if (var_type != IS_UNKNOWN) {
var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED);
}
if (!(var_type & IS_TRACE_REFERENCE)
&& var_type != IS_UNKNOWN
&& (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) {
exit_point = zend_jit_trace_get_exit_point(opline, 0);
exit_addr = zend_jit_trace_get_exit_addr(exit_point);
if (!exit_addr) {
return 0;
}
| IF_NOT_Z_TYPE FCARG1a, var_type, &exit_addr
//var_info = zend_jit_trace_type_to_info_ex(var_type, var_info);
ZEND_ASSERT(var_info & (1 << var_type));
if (var_type < IS_STRING) {
var_info = (1 << var_type);
} else if (var_type != IS_ARRAY) {
var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN));
} else {
var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN));
}
*var_info_ptr = var_info;
}
return 1;
}
static bool zend_jit_may_reuse_reg(const zend_op *opline, const zend_ssa_op *ssa_op, zend_ssa *ssa, int def_var, int use_var)
{
if ((ssa->var_info[def_var].type & ~MAY_BE_GUARD) != (ssa->var_info[use_var].type & ~MAY_BE_GUARD)) {
return 0;
}
switch (opline->opcode) {
case ZEND_QM_ASSIGN:
case ZEND_SEND_VAR:
case ZEND_ASSIGN:
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
return 1;
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
if (def_var == ssa_op->result_def &&
use_var == ssa_op->op1_use) {
return 1;
}
break;
default:
break;
}
return 0;
}
static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_SEND_VAR:
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
return (opline->op2_type != IS_CONST);
case ZEND_QM_ASSIGN:
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
return 1;
case ZEND_RETURN:
return (op_array->type != ZEND_EVAL_CODE && op_array->function_name);
case ZEND_ASSIGN:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return
opline->op1_type == IS_CV &&
!(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_RESOURCE|MAY_BE_OBJECT|MAY_BE_REF)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)));
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_DOUBLE)));
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
case ZEND_SL:
case ZEND_SR:
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
return !((op1_info | op2_info) & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG));
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
op1_info = OP1_INFO();
op2_info = OP1_DEF_INFO();
return opline->op1_type == IS_CV
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF) - MAY_BE_LONG))
&& (op2_info & MAY_BE_LONG);
case ZEND_STRLEN:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING;
case ZEND_COUNT:
op1_info = OP1_INFO();
return (opline->op1_type & (IS_CV|IS_CONST))
&& (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY;
case ZEND_JMPZ:
case ZEND_JMPNZ:
if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) {
if (!ssa->cfg.map) {
return 0;
}
if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start &&
((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) {
return 0;
}
}
ZEND_FALLTHROUGH;
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
return 1;
case ZEND_FETCH_CONSTANT:
return 1;
case ZEND_FETCH_DIM_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (trace
&& trace->op1_type != IS_UNKNOWN
&& (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) {
op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY);
}
return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) &&
(!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || !(op1_info & MAY_BE_RC1)) &&
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) ||
(((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING) &&
(!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & MAY_BE_RC1))));
}
return 0;
}
static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var)
{
if (ssa->vars[var].no_val) {
/* we don't need the value */
return 0;
}
if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) {
/* Disable global register allocation,
* register allocation for SSA variables connected through Phi functions
*/
if (ssa->vars[var].definition_phi) {
return 0;
}
if (ssa->vars[var].phi_use_chain) {
zend_ssa_phi *phi = ssa->vars[var].phi_use_chain;
do {
if (!ssa->vars[phi->ssa_var].no_val) {
return 0;
}
phi = zend_ssa_next_use_phi(ssa, var, phi);
} while (phi);
}
}
if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) &&
((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) {
/* bad type */
return 0;
}
return 1;
}
static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var)
{
if (!zend_jit_var_supports_reg(ssa, var)) {
return 0;
}
if (ssa->vars[var].definition >= 0) {
uint32_t def = ssa->vars[var].definition;
if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) {
return 0;
}
}
if (ssa->vars[var].use_chain >= 0) {
int use = ssa->vars[var].use_chain;
do {
if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) &&
!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) {
return 0;
}
use = zend_ssa_next_use(ssa->ops, var, use);
} while (use >= 0);
}
return 1;
}
static bool zend_needs_extra_reg_for_const(const zend_op *opline, uint8_t op_type, znode_op op)
{
|.if X64
|| if (op_type == IS_CONST) {
|| zval *zv = RT_CONSTANT(opline, op);
|| if (Z_TYPE_P(zv) == IS_DOUBLE && Z_DVAL_P(zv) != 0 && !IS_SIGNED_32BIT(zv)) {
|| return 1;
|| } else if (Z_TYPE_P(zv) == IS_LONG && !IS_SIGNED_32BIT(Z_LVAL_P(zv))) {
|| return 1;
|| }
|| }
|.endif
return 0;
}
static zend_regset zend_jit_get_def_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use)
{
uint32_t op1_info, op2_info;
switch (opline->opcode) {
case ZEND_FETCH_DIM_R:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (((opline->op1_type & (IS_TMP_VAR|IS_VAR)) &&
(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) ||
((opline->op2_type & (IS_TMP_VAR|IS_VAR)) &&
(op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)))) {
return ZEND_REGSET(ZREG_FCARG1);
}
break;
default:
break;
}
return ZEND_REGSET_EMPTY;
}
static zend_regset zend_jit_get_scratch_regset(const zend_op *opline, const zend_ssa_op *ssa_op, const zend_op_array *op_array, zend_ssa *ssa, int current_var, bool last_use)
{
uint32_t op1_info, op2_info, res_info;
zend_regset regset = ZEND_REGSET_SCRATCH;
switch (opline->opcode) {
case ZEND_NOP:
case ZEND_OP_DATA:
case ZEND_JMP:
case ZEND_RETURN:
regset = ZEND_REGSET_EMPTY;
break;
case ZEND_QM_ASSIGN:
if (ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
/* break missing intentionally */
case ZEND_SEND_VAL:
case ZEND_SEND_VAL_EX:
if (opline->op2_type == IS_CONST) {
break;
}
if (ssa_op->op1_use == current_var) {
regset = ZEND_REGSET(ZREG_R0);
break;
}
op1_info = OP1_INFO();
if (!(op1_info & MAY_BE_UNDEF)) {
if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
regset = ZEND_REGSET(ZREG_R0);
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
}
}
break;
case ZEND_SEND_VAR:
if (opline->op2_type == IS_CONST) {
break;
}
if (ssa_op->op1_use == current_var ||
ssa_op->op1_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
if (!(op1_info & MAY_BE_UNDEF)) {
if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
} else if ((op1_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
if (op1_info & MAY_BE_REF) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_ASSIGN:
if (ssa_op->op2_use == current_var ||
ssa_op->op2_def == current_var ||
ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (opline->op1_type == IS_CV
&& !(op2_info & MAY_BE_UNDEF)
&& !(op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) {
if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
} else if ((op2_info & (MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_LONG) {
regset = ZEND_REGSET(ZREG_R0);
} else {
regset = ZEND_REGSET_UNION(ZEND_REGSET(ZREG_R0), ZEND_REGSET(ZREG_R2));
}
}
break;
case ZEND_PRE_INC:
case ZEND_PRE_DEC:
case ZEND_POST_INC:
case ZEND_POST_DEC:
if (ssa_op->op1_use == current_var ||
ssa_op->op1_def == current_var ||
ssa_op->result_def == current_var) {
regset = ZEND_REGSET_EMPTY;
break;
}
op1_info = OP1_INFO();
if (opline->op1_type == IS_CV
&& (op1_info & MAY_BE_LONG)
&& !(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (op1_info & MAY_BE_DOUBLE) {
regset = ZEND_REGSET(ZREG_XMM0);
}
if (opline->result_type != IS_UNUSED && (op1_info & MAY_BE_LONG)) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
break;
case ZEND_ADD:
case ZEND_SUB:
case ZEND_MUL:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) {
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
res_info = RES_INFO();
if (res_info & MAY_BE_DOUBLE) {
ZEND_REGSET_INCL(regset, ZREG_R0);
ZEND_REGSET_INCL(regset, ZREG_XMM0);
ZEND_REGSET_INCL(regset, ZREG_XMM1);
} else if (res_info & MAY_BE_GUARD) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
if (opline->op1_type == IS_CONST) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if (ssa_op->result_def != current_var) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
if (opline->op2_type == IS_CONST) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if (zend_is_commutative(opline->opcode)) {
if (ssa_op->result_def != current_var) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
} else {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_XMM1);
}
}
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use) &&
(!zend_is_commutative(opline->opcode) || ssa_op->op2_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
}
if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) ||
zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) {
if (!ZEND_REGSET_IN(regset, ZREG_R0)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
} else {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_BW_OR:
case ZEND_BW_AND:
case ZEND_BW_XOR:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
regset = ZEND_REGSET_EMPTY;
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) ||
zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) {
if (!ZEND_REGSET_IN(regset, ZREG_R0)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
} else {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_SL:
case ZEND_SR:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
bw_op:
regset = ZEND_REGSET_EMPTY;
if (ssa_op->result_def != current_var &&
(ssa_op->op1_use != current_var || !last_use)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if (opline->op2_type != IS_CONST && ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
break;
case ZEND_MOD:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG)) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-MAY_BE_LONG))) {
if (opline->op2_type == IS_CONST &&
opline->op1_type != IS_CONST &&
Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) == IS_LONG &&
zend_long_is_power_of_two(Z_LVAL_P(RT_CONSTANT(opline, opline->op2))) &&
OP1_HAS_RANGE() &&
OP1_MIN_RANGE() >= 0) {
/* MOD is going to be optimized into AND */
goto bw_op;
} else {
regset = ZEND_REGSET_EMPTY;
ZEND_REGSET_INCL(regset, ZREG_R0);
ZEND_REGSET_INCL(regset, ZREG_R2);
if (opline->op2_type == IS_CONST) {
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
}
break;
case ZEND_IS_SMALLER:
case ZEND_IS_SMALLER_OR_EQUAL:
case ZEND_IS_EQUAL:
case ZEND_IS_NOT_EQUAL:
case ZEND_IS_IDENTICAL:
case ZEND_IS_NOT_IDENTICAL:
case ZEND_CASE:
op1_info = OP1_INFO();
op2_info = OP2_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) &&
!(op2_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (!(opline->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ))) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) &&
opline->op1_type != IS_CONST && opline->op2_type != IS_CONST) {
if (ssa_op->op1_use != current_var &&
ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_DOUBLE)) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_LONG)) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
if ((op1_info & MAY_BE_DOUBLE) && (op2_info & MAY_BE_DOUBLE)) {
if (ssa_op->op1_use != current_var &&
ssa_op->op2_use != current_var) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
}
if (zend_needs_extra_reg_for_const(opline, opline->op1_type, opline->op1) ||
zend_needs_extra_reg_for_const(opline, opline->op2_type, opline->op2)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
break;
case ZEND_BOOL:
case ZEND_BOOL_NOT:
case ZEND_JMPZ:
case ZEND_JMPNZ:
case ZEND_JMPZ_EX:
case ZEND_JMPNZ_EX:
op1_info = OP1_INFO();
if (!(op1_info & ((MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE)))) {
regset = ZEND_REGSET_EMPTY;
if (op1_info & MAY_BE_DOUBLE) {
ZEND_REGSET_INCL(regset, ZREG_XMM0);
}
if (opline->opcode == ZEND_BOOL ||
opline->opcode == ZEND_BOOL_NOT ||
opline->opcode == ZEND_JMPZ_EX ||
opline->opcode == ZEND_JMPNZ_EX) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
}
break;
case ZEND_DO_UCALL:
case ZEND_DO_FCALL:
case ZEND_DO_FCALL_BY_NAME:
case ZEND_INCLUDE_OR_EVAL:
case ZEND_GENERATOR_CREATE:
case ZEND_YIELD:
case ZEND_YIELD_FROM:
regset = ZEND_REGSET_UNION(ZEND_REGSET_GP, ZEND_REGSET_FP);
break;
default:
break;
}
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (ssa_op == ssa->ops
&& JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].op == ZEND_JIT_TRACE_INIT_CALL
&& (JIT_G(current_trace)[ZEND_JIT_TRACE_START_REC_SIZE].info & ZEND_JIT_TRACE_FAKE_INIT_CALL)) {
ZEND_REGSET_INCL(regset, ZREG_R0);
ZEND_REGSET_INCL(regset, ZREG_R1);
}
}
/* %r0 is used to check EG(vm_interrupt) */
if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) {
if (ssa_op == ssa->ops
&& (JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_LOOP ||
JIT_G(current_trace)->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_CALL)) {
#if ZTS
ZEND_REGSET_INCL(regset, ZREG_R0);
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
#endif
}
} else {
uint32_t b = ssa->cfg.map[ssa_op - ssa->ops];
if ((ssa->cfg.blocks[b].flags & ZEND_BB_LOOP_HEADER) != 0
&& ssa->cfg.blocks[b].start == ssa_op - ssa->ops) {
#if ZTS
ZEND_REGSET_INCL(regset, ZREG_R0);
#else
if ((sizeof(void*) == 8 && !IS_SIGNED_32BIT(&EG(vm_interrupt)))) {
ZEND_REGSET_INCL(regset, ZREG_R0);
}
#endif
}
}
return regset;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* indent-tabs-mode: t
* End:
*/
Reference in New Issue View Git Blame Copy Permalink