From 12f9dad185dbbdf0e86f54dabe63b1456ca5a3d3 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Thu, 16 Sep 2021 10:32:08 +0300 Subject: [PATCH] Tracing JIT: Record information about elements of arrays and use it to improve generated code (ASSIGN_DIM). --- ext/opcache/jit/zend_jit.c | 4 +- ext/opcache/jit/zend_jit_arm64.dasc | 22 +++++++---- ext/opcache/jit/zend_jit_internal.h | 3 ++ ext/opcache/jit/zend_jit_trace.c | 35 ++++++++++++++-- ext/opcache/jit/zend_jit_vm_helpers.c | 57 +++++++++++++++++++++++++++ ext/opcache/jit/zend_jit_x86.dasc | 26 +++++++----- 6 files changed, 123 insertions(+), 24 deletions(-) diff --git a/ext/opcache/jit/zend_jit.c b/ext/opcache/jit/zend_jit.c index 1504c022e86..f9e7d856bae 100644 --- a/ext/opcache/jit/zend_jit.c +++ b/ext/opcache/jit/zend_jit.c @@ -3121,7 +3121,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op } if (!zend_jit_assign_dim_op(&dasm_state, opline, OP1_INFO(), OP1_DEF_INFO(), OP1_REG_ADDR(), OP2_INFO(), - OP1_DATA_INFO(), OP1_DATA_RANGE(), + OP1_DATA_INFO(), OP1_DATA_RANGE(), IS_UNKNOWN, zend_may_throw(opline, ssa_op, op_array, ssa))) { goto jit_failure; } @@ -3134,7 +3134,7 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op break; } if (!zend_jit_assign_dim(&dasm_state, opline, - OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), + OP1_INFO(), OP1_REG_ADDR(), OP2_INFO(), OP1_DATA_INFO(), IS_UNKNOWN, zend_may_throw(opline, ssa_op, op_array, ssa))) { goto jit_failure; } diff --git a/ext/opcache/jit/zend_jit_arm64.dasc b/ext/opcache/jit/zend_jit_arm64.dasc index f86369bafc5..6b2d67c9447 100644 --- a/ext/opcache/jit/zend_jit_arm64.dasc +++ b/ext/opcache/jit/zend_jit_arm64.dasc @@ -4882,7 +4882,7 @@ static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); } -static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) +static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) /* Labels: 1,2,3,4,5 */ { zend_jit_addr op2_addr = OP2_ADDR(); @@ -4944,6 +4944,12 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } packed_loaded = 1; } + + if (dim_type == IS_UNDEF && type == BP_VAR_W) { + /* don't generate "fast" code for packed array */ + packed_loaded = 0; + } + if (packed_loaded) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { @@ -5701,7 +5707,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, return 1; } -static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw) +static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) { zend_jit_addr op2_addr, op3_addr, res_addr; @@ -5799,7 +5805,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_REG0, 0); - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { return 0; } @@ -5912,7 +5918,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t return 1; } -static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw) +static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) { zend_jit_addr op2_addr, op3_addr, var_addr; @@ -6016,7 +6022,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 var_info |= MAY_BE_RC1; } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { return 0; } @@ -10915,7 +10921,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } } | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr, TMP1 - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, IS_UNKNOWN, res_exit_addr, not_found_exit_addr, exit_addr)) { return 0; } } @@ -11217,7 +11223,7 @@ static int zend_jit_fetch_dim(dasm_State **Dst, ZEND_UNREACHABLE(); } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, IS_UNKNOWN, NULL, NULL, NULL)) { return 0; } @@ -11338,7 +11344,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, not_found_exit_addr = exit_addr; } } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, IS_UNKNOWN, found_exit_addr, not_found_exit_addr, NULL)) { return 0; } diff --git a/ext/opcache/jit/zend_jit_internal.h b/ext/opcache/jit/zend_jit_internal.h index 0a38573d200..7f27d3d57b5 100644 --- a/ext/opcache/jit/zend_jit_internal.h +++ b/ext/opcache/jit/zend_jit_internal.h @@ -438,6 +438,7 @@ typedef enum _zend_jit_trace_op { ZEND_JIT_TRACE_VM, ZEND_JIT_TRACE_OP1_TYPE, ZEND_JIT_TRACE_OP2_TYPE, + ZEND_JIT_TRACE_VAL_INFO, ZEND_JIT_TRACE_INIT_CALL, ZEND_JIT_TRACE_DO_ICALL, ZEND_JIT_TRACE_ENTER, @@ -451,6 +452,8 @@ typedef enum _zend_jit_trace_op { #define IS_TRACE_REFERENCE (1<<5) #define IS_TRACE_INDIRECT (1<<6) +#define IS_TRACE_TYPE_MASK 0xf + #define ZEND_JIT_TRACE_FAKE_INIT_CALL 0x00000100 #define ZEND_JIT_TRACE_RETURN_VALUE_USED 0x00000100 diff --git a/ext/opcache/jit/zend_jit_trace.c b/ext/opcache/jit/zend_jit_trace.c index 96bb2dac4d4..b19bffce9b8 100644 --- a/ext/opcache/jit/zend_jit_trace.c +++ b/ext/opcache/jit/zend_jit_trace.c @@ -975,7 +975,7 @@ static int find_return_ssa_var(zend_jit_trace_rec *p, zend_ssa_op *ssa_op) } } return -1; - } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) { + } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) { /*skip */ } else { return -1; @@ -1001,7 +1001,7 @@ static const zend_op *zend_jit_trace_find_init_fcall_op(zend_jit_trace_rec *p, c return p->opline; } return NULL; - } else if (p->op == ZEND_JIT_TRACE_OP1_TYPE || p->op == ZEND_JIT_TRACE_OP2_TYPE) { + } else if (p->op >= ZEND_JIT_TRACE_OP1_TYPE && p->op <= ZEND_JIT_TRACE_VAL_INFO) { /*skip */ } else { return NULL; @@ -1597,6 +1597,9 @@ static zend_ssa *zend_jit_trace_build_tssa(zend_jit_trace_rec *trace_buffer, uin op2_ce = (zend_class_entry*)(p+1)->ce; p++; } + if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { + p++; + } switch (opline->opcode) { case ZEND_ASSIGN_OP: @@ -4072,6 +4075,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par uint8_t op3_type = p->op3_type; uint8_t orig_op1_type = op1_type; uint8_t orig_op2_type = op2_type; + uint8_t val_type = IS_UNKNOWN; bool op1_indirect; zend_class_entry *op1_ce = NULL; zend_class_entry *op2_ce = NULL; @@ -4098,6 +4102,10 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op2_ce = (zend_class_entry*)(p+1)->ce; p++; } + if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { + val_type = (p+1)->op1_type; + p++; + } frame_flags = 0; @@ -4387,7 +4395,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op1_def_info = OP1_DEF_INFO(); if (!zend_jit_assign_dim_op(&dasm_state, opline, op1_info, op1_def_info, op1_addr, op2_info, - op1_data_info, OP1_DATA_RANGE(), + op1_data_info, OP1_DATA_RANGE(), val_type, zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) { goto jit_failure; } @@ -4636,7 +4644,7 @@ static const void *zend_jit_trace(zend_jit_trace_rec *trace_buffer, uint32_t par op1_data_info = OP1_DATA_INFO(); CHECK_OP1_DATA_TRACE_TYPE(); if (!zend_jit_assign_dim(&dasm_state, opline, - op1_info, op1_addr, op2_info, op1_data_info, + op1_info, op1_addr, op2_info, op1_data_info, val_type, zend_may_throw_ex(opline, ssa_op, op_array, ssa, op1_info, op2_info))) { goto jit_failure; } @@ -6919,6 +6927,25 @@ static void zend_jit_dump_trace(zend_jit_trace_rec *trace_buffer, zend_ssa *tssa fprintf(stderr, " op3(%s%s)", ref, type); } } + if ((p+1)->op == ZEND_JIT_TRACE_VAL_INFO) { + uint8_t val_type; + const char *type; + + if (op1_type == IS_UNKNOWN && op2_type == IS_UNKNOWN && op3_type == IS_UNKNOWN) { + fprintf(stderr, " ;"); + } + p++; + val_type = p->op1_type; + + if (val_type == IS_UNDEF) { + type = "undef"; + } else if (val_type == IS_REFERENCE) { + type = "ref"; + } else { + type = zend_get_type_by_const(val_type); + } + fprintf(stderr, " val(%s)", type); + } fprintf(stderr, "\n"); idx++; diff --git a/ext/opcache/jit/zend_jit_vm_helpers.c b/ext/opcache/jit/zend_jit_vm_helpers.c index a78fcf4f6f2..8fba54cc9b7 100644 --- a/ext/opcache/jit/zend_jit_vm_helpers.c +++ b/ext/opcache/jit/zend_jit_vm_helpers.c @@ -729,6 +729,63 @@ zend_jit_trace_stop ZEND_FASTCALL zend_jit_trace_execute(zend_execute_data *ex, TRACE_RECORD(ZEND_JIT_TRACE_OP2_TYPE, 0, ce2); } + switch (opline->opcode) { + case ZEND_FETCH_DIM_R: + case ZEND_FETCH_DIM_W: + case ZEND_FETCH_DIM_RW: + case ZEND_FETCH_DIM_IS: + case ZEND_FETCH_DIM_FUNC_ARG: + case ZEND_FETCH_DIM_UNSET: + case ZEND_FETCH_LIST_R: + case ZEND_FETCH_LIST_W: + case ZEND_ASSIGN_DIM: + case ZEND_ASSIGN_DIM_OP: + case ZEND_UNSET_DIM: + case ZEND_ISSET_ISEMPTY_DIM_OBJ: + if (opline->op1_type == IS_CONST) { + zval *arr = RT_CONSTANT(opline, opline->op1); + op1_type = Z_TYPE_P(arr); + } + if ((op1_type & IS_TRACE_TYPE_MASK) == IS_ARRAY + && opline->op2_type != IS_UNDEF) { + zval *arr, *dim, *val; + uint8_t val_type = IS_UNDEF; + + if (opline->op2_type == IS_CONST) { + dim = RT_CONSTANT(opline, opline->op2); + } else { + dim = EX_VAR(opline->op2.var); + } + + if (Z_TYPE_P(dim) == IS_LONG || Z_TYPE_P(dim) == IS_STRING) { + if (opline->op1_type == IS_CONST) { + arr = RT_CONSTANT(opline, opline->op1); + } else { + arr = EX_VAR(opline->op1.var); + } + if (Z_TYPE_P(arr) == IS_INDIRECT) { + arr = Z_INDIRECT_P(arr); + } + if (Z_TYPE_P(arr) == IS_REFERENCE) { + arr = Z_REFVAL_P(arr); + } + ZEND_ASSERT(Z_TYPE_P(arr) == IS_ARRAY); + if (Z_TYPE_P(dim) == IS_LONG) { + val = zend_hash_index_find(Z_ARRVAL_P(arr), Z_LVAL_P(dim)); + } else /*if Z_TYPE_P(dim) == IS_STRING)*/ { + val = zend_hash_find(Z_ARRVAL_P(arr), Z_STR_P(dim)); + } + if (val) { + val_type = Z_TYPE_P(val); + } + TRACE_RECORD_VM(ZEND_JIT_TRACE_VAL_INFO, NULL, val_type, 0, 0); + } + } + break; + default: + break; + } + if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_ICALL || opline->opcode == ZEND_DO_UCALL diff --git a/ext/opcache/jit/zend_jit_x86.dasc b/ext/opcache/jit/zend_jit_x86.dasc index 8a5bcfaa08e..bcec91c83cb 100644 --- a/ext/opcache/jit/zend_jit_x86.dasc +++ b/ext/opcache/jit/zend_jit_x86.dasc @@ -5341,7 +5341,7 @@ static int zend_jit_concat(dasm_State **Dst, const zend_op *opline, uint32_t op1 return zend_jit_concat_helper(Dst, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); } -static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) +static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr) /* Labels: 1,2,3,4,5 */ { zend_jit_addr op2_addr = OP2_ADDR(); @@ -5401,6 +5401,12 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o } packed_loaded = 1; } + + if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) { + /* don't generate "fast" code for packed array */ + packed_loaded = 0; + } + if (packed_loaded) { | // ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { @@ -5578,7 +5584,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o if (packed_loaded) { | IF_NOT_Z_TYPE r0, IS_UNDEF, >8 } - if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded) { +// if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded) { |2: |4: if (!op2_loaded) { @@ -5586,7 +5592,7 @@ static int zend_jit_fetch_dimension_address_inner(dasm_State **Dst, const zend_o | GET_ZVAL_LVAL ZREG_FCARG2, op2_addr } | EXT_CALL zend_hash_index_lookup, r0 - } +// } break; default: ZEND_UNREACHABLE(); @@ -6178,7 +6184,7 @@ static int zend_jit_assign_to_variable(dasm_State **Dst, return 1; } -static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, int may_throw) +static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t val_info, uint8_t dim_type, int may_throw) { zend_jit_addr op2_addr, op3_addr, res_addr; @@ -6274,7 +6280,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_R0, 0); - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, NULL, NULL, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_W, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { return 0; } @@ -6404,7 +6410,7 @@ static int zend_jit_assign_dim(dasm_State **Dst, const zend_op *opline, uint32_t return 1; } -static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, int may_throw) +static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, uint32_t op1_data_info, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) { zend_jit_addr op2_addr, op3_addr, var_addr; @@ -6506,7 +6512,7 @@ static int zend_jit_assign_dim_op(dasm_State **Dst, const zend_op *opline, uint3 var_info |= MAY_BE_RC1; } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, NULL, NULL, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_VAR_RW, op1_info, op2_info, dim_type, NULL, NULL, NULL)) { return 0; } @@ -11553,7 +11559,7 @@ static int zend_jit_fetch_dim_read(dasm_State **Dst, } } | GET_ZVAL_LVAL ZREG_FCARG1, op1_addr - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, res_exit_addr, not_found_exit_addr, exit_addr)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, IS_UNKNOWN, res_exit_addr, not_found_exit_addr, exit_addr)) { return 0; } } @@ -11870,7 +11876,7 @@ static int zend_jit_fetch_dim(dasm_State **Dst, ZEND_UNREACHABLE(); } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, NULL, NULL, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, type, op1_info, op2_info, IS_UNKNOWN, NULL, NULL, NULL)) { return 0; } @@ -12000,7 +12006,7 @@ static int zend_jit_isset_isempty_dim(dasm_State **Dst, not_found_exit_addr = exit_addr; } } - if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, found_exit_addr, not_found_exit_addr, NULL)) { + if (!zend_jit_fetch_dimension_address_inner(Dst, opline, BP_JIT_IS, op1_info, op2_info, IS_UNKNOWN, found_exit_addr, not_found_exit_addr, NULL)) { return 0; }