diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index fa46d93fc67..cf0443bbddc 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -974,7 +974,7 @@ restart: } while (1); ir_fold_restart: - if (!(ctx->flags2 & IR_OPT_IN_SCCP)) { + if (!ctx->use_lists) { op1_insn = ctx->ir_base + op1; op2_insn = ctx->ir_base + op2; op3_insn = ctx->ir_base + op3; @@ -987,7 +987,7 @@ ir_fold_restart: return IR_FOLD_DO_RESTART; } ir_fold_cse: - if (!(ctx->flags2 & IR_OPT_IN_SCCP)) { + if (!ctx->use_lists) { /* Local CSE */ ref = _ir_fold_cse(ctx, opt, op1, op2, op3); if (ref) { @@ -1017,7 +1017,7 @@ ir_fold_cse: return IR_FOLD_DO_CSE; } ir_fold_emit: - if (!(ctx->flags2 & IR_OPT_IN_SCCP)) { + if (!ctx->use_lists) { return ir_emit(ctx, opt, op1, op2, op3); } else { ctx->fold_insn.optx = opt; @@ -1027,14 +1027,14 @@ ir_fold_emit: return IR_FOLD_DO_EMIT; } ir_fold_copy: - if (!(ctx->flags2 & IR_OPT_IN_SCCP)) { + if (!ctx->use_lists) { return ref; } else { ctx->fold_insn.op1 = ref; return IR_FOLD_DO_COPY; } ir_fold_const: - if (!(ctx->flags2 & IR_OPT_IN_SCCP)) { + if (!ctx->use_lists) { return ir_const(ctx, val, IR_OPT_TYPE(opt)); } else { ctx->fold_insn.opt = IR_OPT(IR_OPT_TYPE(opt), IR_OPT_TYPE(opt)); @@ -2015,7 +2015,7 @@ ir_alias ir_check_partial_aliasing(const ir_ctx *ctx, ir_ref addr1, ir_ref addr2 return IR_MAY_ALIAS; } -IR_ALWAYS_INLINE ir_ref ir_find_aliasing_load_i(ir_ctx *ctx, ir_ref ref, ir_type type, ir_ref addr, ir_ref limit, bool allow_casting) +IR_ALWAYS_INLINE ir_ref ir_find_aliasing_load_i(ir_ctx *ctx, ir_ref ref, ir_type type, ir_ref addr, ir_ref limit) { ir_insn *insn; uint32_t modified_regset = 0; @@ -2026,13 +2026,11 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_load_i(ir_ctx *ctx, ir_ref ref, ir_type if (insn->op2 == addr) { if (insn->type == type) { return ref; /* load forwarding (L2L) */ - } else if (!allow_casting) { - /* pass */ } else if (ir_type_size[insn->type] == ir_type_size[type]) { - return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), ref); /* load forwarding with bitcast (L2L) */ + return ref; /* load forwarding with bitcast (L2L) */ } else if (ir_type_size[insn->type] > ir_type_size[type] && IR_IS_TYPE_INT(type) && IR_IS_TYPE_INT(insn->type)) { - return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), ref); /* partial load forwarding (L2L) */ + return ref; /* partial load forwarding (L2L) */ } } } else if (insn->op == IR_STORE) { @@ -2045,13 +2043,11 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_load_i(ir_ctx *ctx, ir_ref ref, ir_type return IR_UNUSED; } else if (type2 == type) { return insn->op3; /* store forwarding (S2L) */ - } else if (!allow_casting) { - return IR_UNUSED; } else if (ir_type_size[type2] == ir_type_size[type]) { - return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), insn->op3); /* store forwarding with bitcast (S2L) */ + return insn->op3; /* store forwarding with bitcast (S2L) */ } else if (ir_type_size[type2] > ir_type_size[type] && IR_IS_TYPE_INT(type) && IR_IS_TYPE_INT(type2)) { - return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), insn->op3); /* partial store forwarding (S2L) */ + return insn->op3; /* partial store forwarding (S2L) */ } else { return IR_UNUSED; } @@ -2071,10 +2067,10 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_load_i(ir_ctx *ctx, ir_ref ref, ir_type ir_ref ir_find_aliasing_load(ir_ctx *ctx, ir_ref ref, ir_type type, ir_ref addr) { - return ir_find_aliasing_load_i(ctx, ref, type, addr, (addr > 0 && addr < ref) ? addr : 1, 0); + return ir_find_aliasing_load_i(ctx, ref, type, addr, (addr > 0 && addr < ref) ? addr : 1); } -IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vload_i(ir_ctx *ctx, ir_ref ref, ir_type type, ir_ref var, bool allow_casting) +IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vload_i(ir_ctx *ctx, ir_ref ref, ir_type type, ir_ref var) { ir_insn *insn; @@ -2084,8 +2080,6 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vload_i(ir_ctx *ctx, ir_ref ref, ir_typ if (insn->op2 == var) { if (insn->type == type) { return ref; /* load forwarding (L2L) */ - } else if (!allow_casting) { - /* pass */; } else if (ir_type_size[insn->type] == ir_type_size[type]) { return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), ref); /* load forwarding with bitcast (L2L) */ } else if (ir_type_size[insn->type] > ir_type_size[type] @@ -2099,8 +2093,6 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vload_i(ir_ctx *ctx, ir_ref ref, ir_typ if (insn->op2 == var) { if (type2 == type) { return insn->op3; /* store forwarding (S2L) */ - } else if (!allow_casting) { - break; } else if (ir_type_size[type2] == ir_type_size[type]) { return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), insn->op3); /* store forwarding with bitcast (S2L) */ } else if (ir_type_size[type2] > ir_type_size[type] @@ -2121,7 +2113,7 @@ IR_ALWAYS_INLINE ir_ref ir_find_aliasing_vload_i(ir_ctx *ctx, ir_ref ref, ir_typ ir_ref ir_find_aliasing_vload(ir_ctx *ctx, ir_ref ref, ir_type type, ir_ref var) { - return ir_find_aliasing_vload_i(ctx, ref, type, var, 0); + return ir_find_aliasing_vload_i(ctx, ref, type, var); } IR_ALWAYS_INLINE ir_ref ir_find_aliasing_store_i(ir_ctx *ctx, ir_ref ref, ir_ref addr, ir_ref val, ir_ref limit) @@ -3120,16 +3112,23 @@ void _ir_AFREE(ir_ctx *ctx, ir_ref size) ir_ref _ir_VLOAD(ir_ctx *ctx, ir_type type, ir_ref var) { - ir_ref ref = IR_UNUSED; + ir_ref ref; IR_ASSERT(ctx->control); if (EXPECTED(ctx->flags & IR_OPT_FOLDING)) { - ref = ir_find_aliasing_vload_i(ctx, ctx->control, type, var, 1); + ref = ir_find_aliasing_vload_i(ctx, ctx->control, type, var); + if (ref) { + ir_insn *insn = &ctx->ir_base[ref]; + if (insn->type == type) { + return ref; + } else if (ir_type_size[insn->type] == ir_type_size[type]) { + return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), ref); /* load forwarding with bitcast (L2L) */ + } else { + return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), ref); /* partial load forwarding (L2L) */ + } + } } - if (!ref) { - ctx->control = ref = ir_emit2(ctx, IR_OPT(IR_VLOAD, type), ctx->control, var); - } - return ref; + return ctx->control = ir_emit2(ctx, IR_OPT(IR_VLOAD, type), ctx->control, var); } void _ir_VSTORE(ir_ctx *ctx, ir_ref var, ir_ref val) @@ -3164,22 +3163,36 @@ void _ir_RSTORE(ir_ctx *ctx, ir_ref reg, ir_ref val) ir_ref _ir_LOAD(ir_ctx *ctx, ir_type type, ir_ref addr) { - ir_ref ref = IR_UNUSED; + ir_ref ref; IR_ASSERT(ctx->control); if (EXPECTED(ctx->flags & IR_OPT_FOLDING)) { - ref = ir_find_aliasing_load_i(ctx, ctx->control, type, addr, (addr > 0) ? addr : 1, 1); + if (ctx->ir_base[addr].op == IR_VADDR) { + return _ir_VLOAD(ctx, type, ctx->ir_base[addr].op1); + } + ref = ir_find_aliasing_load_i(ctx, ctx->control, type, addr, (addr > 0) ? addr : 1); + if (ref) { + ir_insn *insn = &ctx->ir_base[ref]; + if (insn->type == type) { + return ref; + } else if (ir_type_size[insn->type] == ir_type_size[type]) { + return ir_fold1(ctx, IR_OPT(IR_BITCAST, type), ref); /* load forwarding with bitcast (L2L) */ + } else { + return ir_fold1(ctx, IR_OPT(IR_TRUNC, type), ref); /* partial load forwarding (L2L) */ + } + } } - if (!ref) { - ctx->control = ref = ir_emit2(ctx, IR_OPT(IR_LOAD, type), ctx->control, addr); - } - return ref; + return ctx->control = ir_emit2(ctx, IR_OPT(IR_LOAD, type), ctx->control, addr); } void _ir_STORE(ir_ctx *ctx, ir_ref addr, ir_ref val) { IR_ASSERT(ctx->control); if (EXPECTED(ctx->flags & IR_OPT_FOLDING)) { + if (ctx->ir_base[addr].op == IR_VADDR) { + _ir_VSTORE(ctx, ctx->ir_base[addr].op1, val); + return; + } if (ir_find_aliasing_store_i(ctx, ctx->control, addr, val, (addr > 0) ? addr : 1)) { /* dead STORE */ return; diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 688dccda68e..60c501d0bd4 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -960,9 +960,7 @@ IR_ALWAYS_INLINE void *ir_jit_compile(ir_ctx *ctx, int opt_level, size_t *size) || !ir_mem2ssa(ctx)) { return NULL; } - if (opt_level > 1) { - ir_reset_cfg(ctx); - } + ir_reset_cfg(ctx); } if (opt_level > 1) { diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 42ff430accc..6f0bfea4782 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -1469,6 +1469,27 @@ IR_FOLD(EQ(SEXT, C_ADDR)) op1 = op1_insn->op1; op2 = IR_UNUSED; IR_FOLD_RESTART; + } else { + ir_type type = ctx->ir_base[op1_insn->op1].type; + + if (IR_IS_TYPE_SIGNED(type)) { + switch (ir_type_size[type]) { + case 1: val.i64 = op2_insn->val.i8; break; + case 2: val.i64 = op2_insn->val.i16; break; + case 4: val.i64 = op2_insn->val.i32; break; + default: val.u64 = op2_insn->val.u64; break; + } + } else { + switch (ir_type_size[type]) { + case 1: val.u64 = op2_insn->val.u8; break; + case 2: val.u64 = op2_insn->val.u16; break; + case 4: val.u64 = op2_insn->val.u32; break; + default: val.u64 = op2_insn->val.u64; break; + } + } + op1 = op1_insn->op1; + op2 = ir_const(ctx, val, type); + IR_FOLD_RESTART; } IR_FOLD_NEXT; } @@ -1490,6 +1511,27 @@ IR_FOLD(NE(SEXT, C_ADDR)) { if (op2_insn->val.u64 == 0 && ctx->ir_base[op1_insn->op1].type == IR_BOOL) { IR_FOLD_COPY(op1_insn->op1); + } else { + ir_type type = ctx->ir_base[op1_insn->op1].type; + + if (IR_IS_TYPE_SIGNED(type)) { + switch (ir_type_size[type]) { + case 1: val.i64 = op2_insn->val.i8; break; + case 2: val.i64 = op2_insn->val.i16; break; + case 4: val.i64 = op2_insn->val.i32; break; + default: val.u64 = op2_insn->val.u64; break; + } + } else { + switch (ir_type_size[type]) { + case 1: val.u64 = op2_insn->val.u8; break; + case 2: val.u64 = op2_insn->val.u16; break; + case 4: val.u64 = op2_insn->val.u32; break; + default: val.u64 = op2_insn->val.u64; break; + } + } + op1 = op1_insn->op1; + op2 = ir_const(ctx, val, type); + IR_FOLD_RESTART; } IR_FOLD_NEXT; } @@ -1586,6 +1628,24 @@ IR_FOLD(SUB_OV(_, C_ADDR)) IR_FOLD_NEXT; } +/* This rule is useful for ADD(0, SYM) => SYM */ +IR_FOLD(ADD(C_U8, _)) +IR_FOLD(ADD(C_U16, _)) +IR_FOLD(ADD(C_U32, _)) +IR_FOLD(ADD(C_U64, _)) +IR_FOLD(ADD(C_I8, _)) +IR_FOLD(ADD(C_I16, _)) +IR_FOLD(ADD(C_I32, _)) +IR_FOLD(ADD(C_I64, _)) +IR_FOLD(ADD(C_ADDR, _)) +{ + if (op1_insn->val.u64 == 0) { + /* 0 + a => a */ + IR_FOLD_COPY(op2); + } + IR_FOLD_NEXT; +} + IR_FOLD(SUB(C_I8, _)) IR_FOLD(SUB(C_I16, _)) IR_FOLD(SUB(C_I32, _)) diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 2d0a46bf4b6..0c69b530b02 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -3009,9 +3009,10 @@ remove_mem_insn: } else if (insn->op == IR_LOAD) { val = ir_find_aliasing_load(ctx, insn->op1, insn->type, insn->op2); if (val) { + ir_insn *val_insn; ir_ref prev, next; -remove_load: +remove_aliased_load: prev = insn->op1; next = ir_next_control(ctx, i); ctx->ir_base[next].op1 = prev; @@ -3019,7 +3020,30 @@ remove_load: ir_use_list_replace_one(ctx, prev, i, next); insn->op1 = IR_UNUSED; - ir_iter_replace_insn(ctx, i, val, worklist); + val_insn = &ctx->ir_base[val]; + if (val_insn->type == insn->type) { + ir_iter_replace_insn(ctx, i, val, worklist); + } else { + IR_ASSERT(!IR_IS_CONST_REF(insn->op2)); + ir_use_list_remove_one(ctx, insn->op2, i); + if (ir_is_dead(ctx, insn->op2)) { + /* schedule DCE */ + ir_bitqueue_add(worklist, insn->op2); + } + if (!IR_IS_CONST_REF(val)) { + ir_use_list_add(ctx, val, i); + } + if (ir_type_size[val_insn->type] == ir_type_size[insn->type]) { + /* load forwarding with bitcast (L2L) */ + insn->optx = IR_OPTX(IR_BITCAST, insn->type, 1); + } else { + /* partial load forwarding (L2L) */ + insn->optx = IR_OPTX(IR_TRUNC, insn->type, 1); + } + insn->op1 = val; + insn->op2 = IR_UNUSED; + ir_bitqueue_add(worklist, i); + } } } else if (insn->op == IR_STORE) { if (ir_find_aliasing_store(ctx, insn->op1, insn->op2, insn->op3)) { @@ -3049,7 +3073,7 @@ remove_bitcast: } else if (insn->op == IR_VLOAD) { val = ir_find_aliasing_vload(ctx, insn->op1, insn->type, insn->op2); if (val) { - goto remove_load; + goto remove_aliased_load; } } else if (insn->op == IR_VSTORE) { if (ir_find_aliasing_vstore(ctx, insn->op1, insn->op2, insn->op3)) { @@ -3080,13 +3104,14 @@ int ir_sccp(ir_ctx *ctx) ir_bitqueue sccp_worklist, iter_worklist; ir_insn *_values; - ctx->flags2 |= IR_OPT_IN_SCCP; ir_bitqueue_init(&iter_worklist, ctx->insns_count); ir_bitqueue_init(&sccp_worklist, ctx->insns_count); _values = ir_mem_calloc(ctx->insns_count, sizeof(ir_insn)); + ctx->flags2 |= IR_OPT_IN_SCCP; ir_sccp_analyze(ctx, _values, &sccp_worklist, &iter_worklist); ir_sccp_transform(ctx, _values, &sccp_worklist, &iter_worklist); + ctx->flags2 &= ~IR_OPT_IN_SCCP; ir_mem_free(_values); ir_bitqueue_free(&sccp_worklist); @@ -3097,7 +3122,5 @@ int ir_sccp(ir_ctx *ctx) ir_bitqueue_free(&iter_worklist); - ctx->flags2 &= ~IR_OPT_IN_SCCP; - return 1; }