From 81593cfc6a08e56df836c6382e9873191b9cd6c1 Mon Sep 17 00:00:00 2001 From: Dmitry Stogov Date: Mon, 2 Jun 2025 09:23:39 +0300 Subject: [PATCH] Update IR IR commit: e4343be0082897510c40a1b57baff427c6858878 --- ext/opcache/jit/ir/ir.c | 16 ++++++- ext/opcache/jit/ir/ir.h | 38 ++++++++++------ ext/opcache/jit/ir/ir_cfg.c | 50 ++++++++++++++++++++- ext/opcache/jit/ir/ir_emit.c | 4 +- ext/opcache/jit/ir/ir_fold.h | 78 ++++++++++++++++++++++++++++++++- ext/opcache/jit/ir/ir_private.h | 6 +-- ext/opcache/jit/ir/ir_sccp.c | 23 +++++++--- ext/opcache/jit/ir/ir_x86.dasc | 4 +- 8 files changed, 189 insertions(+), 30 deletions(-) diff --git a/ext/opcache/jit/ir/ir.c b/ext/opcache/jit/ir/ir.c index d9f7e3d0f78..a9f55cc0e46 100644 --- a/ext/opcache/jit/ir/ir.c +++ b/ext/opcache/jit/ir/ir.c @@ -918,7 +918,7 @@ static ir_ref _ir_fold_cse(ir_ctx *ctx, uint32_t opt, ir_ref op1, ir_ref op2, ir #define IR_FOLD_EMIT goto ir_fold_emit #define IR_FOLD_NEXT break -#include "ir_fold_hash.h" +#include #define IR_FOLD_RULE(x) ((x) >> 21) #define IR_FOLD_KEY(x) ((x) & 0x1fffff) @@ -1485,6 +1485,18 @@ void ir_update_op(ir_ctx *ctx, ir_ref ref, uint32_t idx, ir_ref new_val) void ir_array_grow(ir_array *a, uint32_t size) { IR_ASSERT(size > a->size); + if (size >= 256) { + size = IR_ALIGNED_SIZE(size, 256); + } else { + /* Use big enough power of 2 */ + size -= 1; + size |= (size >> 1); + size |= (size >> 2); + size |= (size >> 4); +// size |= (size >> 8); +// size |= (size >> 16); + size += 1; + } a->refs = ir_mem_realloc(a->refs, size * sizeof(ir_ref)); a->size = size; } @@ -1820,7 +1832,7 @@ int ir_mem_flush(void *ptr, size_t size) #else void *ir_mem_mmap(size_t size) { - int prot_flags = PROT_EXEC; + int prot_flags = PROT_EXEC; #if defined(__NetBSD__) prot_flags |= PROT_MPROTECT(PROT_READ|PROT_WRITE); #endif diff --git a/ext/opcache/jit/ir/ir.h b/ext/opcache/jit/ir/ir.h index 60c501d0bd4..ec5e57129c9 100644 --- a/ext/opcache/jit/ir/ir.h +++ b/ext/opcache/jit/ir/ir.h @@ -154,19 +154,27 @@ typedef enum _ir_type { } ir_type; #ifdef IR_64 -# define IR_SIZE_T IR_U64 -# define IR_SSIZE_T IR_I64 -# define IR_UINTPTR_T IR_U64 -# define IR_INTPTR_T IR_I64 -# define IR_C_UINTPTR IR_U64 -# define IR_C_INTPTR IR_I64 +# define IR_SIZE_T IR_U64 +# define IR_SSIZE_T IR_I64 +# define IR_UINTPTR_T IR_U64 +# define IR_INTPTR_T IR_I64 +# define IR_C_UINTPTR IR_U64 +# define IR_C_INTPTR IR_I64 +# define ir_const_size_t ir_const_u64 +# define ir_const_ssize_t ir_const_i64 +# define ir_const_uintptr_t ir_const_u64 +# define ir_const_intptr_t ir_const_i64 #else -# define IR_SIZE_T IR_U32 -# define IR_SSIZE_T IR_I32 -# define IR_UINTPTR_T IR_U32 -# define IR_INTPTR_T IR_I32 -# define IR_C_UINTPTR IR_U32 -# define IR_C_INTPTR IR_I32 +# define IR_SIZE_T IR_U32 +# define IR_SSIZE_T IR_I32 +# define IR_UINTPTR_T IR_U32 +# define IR_INTPTR_T IR_I32 +# define IR_C_UINTPTR IR_U32 +# define IR_C_INTPTR IR_I32 +# define ir_const_size_t ir_const_u32 +# define ir_const_ssize_t ir_const_i32 +# define ir_const_uintptr_t ir_const_u32 +# define ir_const_intptr_t ir_const_i32 #endif /* List of IR opcodes @@ -401,8 +409,10 @@ typedef int32_t ir_ref; #define IR_CONSTS_LIMIT_MIN (-(IR_TRUE - 1)) #define IR_INSNS_LIMIT_MIN (IR_UNUSED + 1) +/* ADDR_MEMBER is neccessary to workaround MSVC C preprocessor bug */ #ifndef IR_64 -# define ADDR_MEMBER uintptr_t addr; +# define ADDR_MEMBER uintptr_t addr; \ + void *ptr; #else # define ADDR_MEMBER #endif @@ -412,6 +422,7 @@ typedef union _ir_val { int64_t i64; #ifdef IR_64 uintptr_t addr; + void *ptr; #endif IR_STRUCT_LOHI( union { @@ -466,6 +477,7 @@ typedef struct _ir_insn { }, union { ir_ref op1; + ir_ref ref; ir_ref prev_const; } ); diff --git a/ext/opcache/jit/ir/ir_cfg.c b/ext/opcache/jit/ir/ir_cfg.c index 16facae51b1..01532c8ea3e 100644 --- a/ext/opcache/jit/ir/ir_cfg.c +++ b/ext/opcache/jit/ir/ir_cfg.c @@ -77,6 +77,51 @@ void ir_reset_cfg(ir_ctx *ctx) } } +static uint32_t IR_NEVER_INLINE ir_cfg_remove_dead_inputs(ir_ctx *ctx, uint32_t *_blocks, ir_block *blocks, uint32_t bb_count) +{ + uint32_t b, count = 0; + ir_block *bb = blocks + 1; + ir_insn *insn; + ir_ref i, j, n, *ops, input; + + for (b = 1; b <= bb_count; b++, bb++) { + bb->successors = count; + count += ctx->use_lists[bb->end].count; + bb->successors_count = 0; + bb->predecessors = count; + insn = &ctx->ir_base[bb->start]; + if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) { + n = insn->inputs_count; + ops = insn->ops; + for (i = 1, j = 1; i <= n; i++) { + input = ops[i]; + if (_blocks[input]) { + if (i != j) { + ops[j] = ops[i]; + } + j++; + } else if (input > 0) { + ir_use_list_remove_one(ctx, input, bb->start); + } + } + j--; + if (j != n) { + if (j == 1) { + insn->op = IR_BEGIN; + } + insn->inputs_count = j; + bb->predecessors_count = j; + j++; + for (;j <= n; j++) { + ops[j] = IR_UNUSED; + } + } + } + count += bb->predecessors_count; + } + return count; +} + int ir_build_cfg(ir_ctx *ctx) { ir_ref n, *p, ref, start, end; @@ -239,7 +284,10 @@ int ir_build_cfg(ir_ctx *ctx) bb++; } IR_BITSET_FOREACH_END(); bb_count = b - 1; - IR_ASSERT(count == edges_count * 2); + if (UNEXPECTED(count != edges_count * 2)) { + count = ir_cfg_remove_dead_inputs(ctx, _blocks, blocks, bb_count); + IR_ASSERT(count != edges_count * 2); + } ir_mem_free(bb_starts); /* Create an array of successor/predecessors control edges */ diff --git a/ext/opcache/jit/ir/ir_emit.c b/ext/opcache/jit/ir/ir_emit.c index 5cf44a51d0f..c82655daf48 100644 --- a/ext/opcache/jit/ir/ir_emit.c +++ b/ext/opcache/jit/ir/ir_emit.c @@ -415,9 +415,9 @@ static int ir_const_label(ir_ctx *ctx, ir_ref ref) } #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) -# include "ir_emit_x86.h" +# include #elif defined(IR_TARGET_AARCH64) -# include "ir_emit_aarch64.h" +# include #else # error "Unknown IR target" #endif diff --git a/ext/opcache/jit/ir/ir_fold.h b/ext/opcache/jit/ir/ir_fold.h index 78f3ca0c01e..88539e52ab0 100644 --- a/ext/opcache/jit/ir/ir_fold.h +++ b/ext/opcache/jit/ir/ir_fold.h @@ -486,10 +486,36 @@ IR_FOLD(MUL(C_FLOAT, C_FLOAT)) } IR_FOLD(DIV(C_U8, C_U8)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op2_insn->val.u64 == 0) { + /* division by zero */ + IR_FOLD_EMIT; + } + IR_FOLD_CONST_U(op1_insn->val.u8 / op2_insn->val.u8); +} + IR_FOLD(DIV(C_U16, C_U16)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op2_insn->val.u64 == 0) { + /* division by zero */ + IR_FOLD_EMIT; + } + IR_FOLD_CONST_U(op1_insn->val.u16 / op2_insn->val.u16); +} + IR_FOLD(DIV(C_U32, C_U32)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op2_insn->val.u64 == 0) { + /* division by zero */ + IR_FOLD_EMIT; + } + IR_FOLD_CONST_U(op1_insn->val.u32 / op2_insn->val.u32); +} + IR_FOLD(DIV(C_U64, C_U64)) -IR_FOLD(DIV(C_ADDR, C_ADDR)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); if (op2_insn->val.u64 == 0) { @@ -499,9 +525,46 @@ IR_FOLD(DIV(C_ADDR, C_ADDR)) IR_FOLD_CONST_U(op1_insn->val.u64 / op2_insn->val.u64); } +IR_FOLD(DIV(C_ADDR, C_ADDR)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op2_insn->val.u64 == 0) { + /* division by zero */ + IR_FOLD_EMIT; + } + IR_FOLD_CONST_U(op1_insn->val.addr / op2_insn->val.addr); +} + IR_FOLD(DIV(C_I8, C_I8)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op2_insn->val.i64 == 0) { + /* division by zero */ + IR_FOLD_EMIT; + } + IR_FOLD_CONST_I(op1_insn->val.i8 / op2_insn->val.i8); +} + IR_FOLD(DIV(C_I16, C_I16)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op2_insn->val.i64 == 0) { + /* division by zero */ + IR_FOLD_EMIT; + } + IR_FOLD_CONST_I(op1_insn->val.i16 / op2_insn->val.i16); +} + IR_FOLD(DIV(C_I32, C_I32)) +{ + IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); + if (op2_insn->val.i64 == 0) { + /* division by zero */ + IR_FOLD_EMIT; + } + IR_FOLD_CONST_I(op1_insn->val.i32 / op2_insn->val.i32); +} + IR_FOLD(DIV(C_I64, C_I64)) { IR_ASSERT(IR_OPT_TYPE(opt) == op1_insn->type); @@ -1135,6 +1198,7 @@ IR_FOLD(MAX(C_FLOAT, C_FLOAT)) IR_FOLD(SEXT(C_I8)) IR_FOLD(SEXT(C_U8)) IR_FOLD(SEXT(C_BOOL)) +IR_FOLD(SEXT(C_CHAR)) { IR_ASSERT(IR_IS_TYPE_INT(IR_OPT_TYPE(opt))); IR_ASSERT(ir_type_size[IR_OPT_TYPE(opt)] > ir_type_size[op1_insn->type]); @@ -1160,6 +1224,7 @@ IR_FOLD(SEXT(C_U32)) IR_FOLD(ZEXT(C_I8)) IR_FOLD(ZEXT(C_U8)) IR_FOLD(ZEXT(C_BOOL)) +IR_FOLD(ZEXT(C_CHAR)) { IR_ASSERT(IR_IS_TYPE_INT(IR_OPT_TYPE(opt))); IR_ASSERT(ir_type_size[IR_OPT_TYPE(opt)] > ir_type_size[op1_insn->type]); @@ -1195,12 +1260,14 @@ IR_FOLD(TRUNC(C_U64)) default: IR_ASSERT(0); case IR_I8: + case IR_CHAR: IR_FOLD_CONST_I(op1_insn->val.i8); case IR_I16: IR_FOLD_CONST_I(op1_insn->val.i16); case IR_I32: IR_FOLD_CONST_I(op1_insn->val.i32); case IR_U8: + case IR_BOOL: IR_FOLD_CONST_U(op1_insn->val.u8); case IR_U16: IR_FOLD_CONST_U(op1_insn->val.u16); @@ -1474,6 +1541,10 @@ IR_FOLD(EQ(SEXT, C_ADDR)) } else { ir_type type = ctx->ir_base[op1_insn->op1].type; + if (op1_insn->op == IR_ZEXT + && (op2_insn->val.u64 >> (ir_type_size[type] * 8)) != 0) { + IR_FOLD_NEXT; + } if (IR_IS_TYPE_SIGNED(type)) { switch (ir_type_size[type]) { case 1: val.i64 = op2_insn->val.i8; break; @@ -1493,6 +1564,7 @@ IR_FOLD(EQ(SEXT, C_ADDR)) op2 = ir_const(ctx, val, type); IR_FOLD_RESTART; } + IR_FOLD_NEXT; } @@ -1518,6 +1590,10 @@ IR_FOLD(NE(SEXT, C_ADDR)) } else { ir_type type = ctx->ir_base[op1_insn->op1].type; + if (op1_insn->op == IR_ZEXT + && (op2_insn->val.u64 >> (ir_type_size[type] * 8)) != 0) { + IR_FOLD_NEXT; + } if (IR_IS_TYPE_SIGNED(type)) { switch (ir_type_size[type]) { case 1: val.i64 = op2_insn->val.i8; break; diff --git a/ext/opcache/jit/ir/ir_private.h b/ext/opcache/jit/ir/ir_private.h index 9c69d6074de..69a0101d24e 100644 --- a/ext/opcache/jit/ir/ir_private.h +++ b/ext/opcache/jit/ir/ir_private.h @@ -255,9 +255,7 @@ IR_ALWAYS_INLINE void ir_arena_free(ir_arena *arena) IR_ALWAYS_INLINE void* ir_arena_alloc(ir_arena **arena_ptr, size_t size) { ir_arena *arena = *arena_ptr; - char *ptr = arena->ptr; - - size = IR_ALIGNED_SIZE(size, 8); + char *ptr = (char*)IR_ALIGNED_SIZE((uintptr_t)arena->ptr, 8); if (EXPECTED(size <= (size_t)(arena->end - ptr))) { arena->ptr = ptr + size; @@ -283,7 +281,7 @@ IR_ALWAYS_INLINE void* ir_arena_checkpoint(ir_arena *arena) return arena->ptr; } -IR_ALWAYS_INLINE void ir_release(ir_arena **arena_ptr, void *checkpoint) +IR_ALWAYS_INLINE void ir_arena_release(ir_arena **arena_ptr, void *checkpoint) { ir_arena *arena = *arena_ptr; diff --git a/ext/opcache/jit/ir/ir_sccp.c b/ext/opcache/jit/ir/ir_sccp.c index 8480861f91f..2e006516df8 100644 --- a/ext/opcache/jit/ir/ir_sccp.c +++ b/ext/opcache/jit/ir/ir_sccp.c @@ -1994,10 +1994,16 @@ static bool ir_try_promote_induction_var_ext(ir_ctx *ctx, ir_ref ext_ref, ir_ref if (use_insn->op >= IR_EQ && use_insn->op <= IR_UGT) { if (use_insn->op1 == phi_ref) { + if (IR_IS_TYPE_SIGNED(type) != IR_IS_TYPE_SIGNED(ctx->ir_base[use_insn->op2].type)) { + return 0; + } if (ir_is_cheaper_ext(ctx, use_insn->op2, ctx->ir_base[phi_ref].op1, ext_ref, op)) { continue; } } else if (use_insn->op2 == phi_ref) { + if (IR_IS_TYPE_SIGNED(type) != IR_IS_TYPE_SIGNED(ctx->ir_base[use_insn->op1].type)) { + return 0; + } if (ir_is_cheaper_ext(ctx, use_insn->op1, ctx->ir_base[phi_ref].op1, ext_ref, op)) { continue; } @@ -2027,10 +2033,16 @@ static bool ir_try_promote_induction_var_ext(ir_ctx *ctx, ir_ref ext_ref, ir_ref if (use_insn->op >= IR_EQ && use_insn->op <= IR_UGT) { if (use_insn->op1 == phi_ref) { + if (IR_IS_TYPE_SIGNED(type) != IR_IS_TYPE_SIGNED(ctx->ir_base[use_insn->op2].type)) { + return 0; + } if (ir_is_cheaper_ext(ctx, use_insn->op2, ctx->ir_base[phi_ref].op1, ext_ref, op)) { continue; } } else if (use_insn->op2 == phi_ref) { + if (IR_IS_TYPE_SIGNED(type) != IR_IS_TYPE_SIGNED(ctx->ir_base[use_insn->op1].type)) { + return 0; + } if (ir_is_cheaper_ext(ctx, use_insn->op1, ctx->ir_base[phi_ref].op1, ext_ref, op)) { continue; } @@ -3570,11 +3582,12 @@ remove_aliased_load: if (val_insn->type == insn->type) { ir_iter_replace_insn(ctx, i, val, worklist); } else { - IR_ASSERT(!IR_IS_CONST_REF(insn->op2)); - ir_use_list_remove_one(ctx, insn->op2, i); - if (ir_is_dead(ctx, insn->op2)) { - /* schedule DCE */ - ir_bitqueue_add(worklist, insn->op2); + if (!IR_IS_CONST_REF(insn->op2)) { + ir_use_list_remove_one(ctx, insn->op2, i); + if (ir_is_dead(ctx, insn->op2)) { + /* schedule DCE */ + ir_bitqueue_add(worklist, insn->op2); + } } if (!IR_IS_CONST_REF(val)) { ir_use_list_add(ctx, val, i); diff --git a/ext/opcache/jit/ir/ir_x86.dasc b/ext/opcache/jit/ir/ir_x86.dasc index d01a8c41359..76602c2b4bc 100644 --- a/ext/opcache/jit/ir/ir_x86.dasc +++ b/ext/opcache/jit/ir/ir_x86.dasc @@ -3236,7 +3236,7 @@ static void ir_emit_store_mem_int_const(ir_ctx *ctx, ir_type type, ir_mem mem, i val = (int64_t)(intptr_t)ir_sym_val(ctx, val_insn); } - if (sizeof(void*) == 4 || IR_IS_SIGNED_32BIT(val)) { + if (ir_type_size[val_insn->type] <= 4 || IR_IS_SIGNED_32BIT(val)) { if (is_arg && ir_type_size[type] < 4) { type = IR_U32; } @@ -3664,7 +3664,7 @@ static int32_t ir_fuse_imm(ir_ctx *ctx, ir_ref ref) IR_ASSERT(IR_IS_SIGNED_32BIT((intptr_t)addr)); return (int32_t)(intptr_t)addr; } else { - IR_ASSERT(IR_IS_SIGNED_32BIT(val_insn->val.i32)); + IR_ASSERT(ir_type_size[val_insn->type] == 4 || IR_IS_SIGNED_32BIT(val_insn->val.i64)); return val_insn->val.i32; } }