Update IR

IR commit: 0108cdf808d1e7dd6b702738949e095151f49040

ext/opcache/jit/ir/ir.c

@@ -23,6 +23,9 @@
 # if defined(__linux__) || defined(__sun)
 #  include <alloca.h>
 # endif
+# if defined(__APPLE__) && defined(__aarch64__)
+#  include <libkern/OSCacheControl.h>
+# endif
 #else
 # define WIN32_LEAN_AND_MEAN
 # include <windows.h>
@@ -109,6 +112,8 @@ static void ir_print_escaped_str(const char *s, size_t len, FILE *f)
 
 void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted)
 {
+	char buf[128];
+
 	if (insn->op == IR_FUNC || insn->op == IR_SYM) {
 		fprintf(f, "%s", ir_get_str(ctx, insn->val.name));
 		return;
@@ -182,14 +187,28 @@ void ir_print_const(const ir_ctx *ctx, const ir_insn *insn, FILE *f, bool quoted
 			if (isnan(insn->val.d)) {
 				fprintf(f, "nan");
 			} else {
-				fprintf(f, "%g", insn->val.d);
+				sprintf(buf, "%g", insn->val.d);
+				if (strtod(buf, NULL) != insn->val.d) {
+					sprintf(buf, "%.53e", insn->val.d);
+					if (strtod(buf, NULL) != insn->val.d) {
+						IR_ASSERT(0 && "can't format double");
+					}
+				}
+				fprintf(f, "%s", buf);
 			}
 			break;
 		case IR_FLOAT:
 			if (isnan(insn->val.f)) {
 				fprintf(f, "nan");
 			} else {
-				fprintf(f, "%g", insn->val.f);
+				sprintf(buf, "%g", insn->val.f);
+				if (strtod(buf, NULL) != insn->val.f) {
+					sprintf(buf, "%.24e", insn->val.f);
+					if (strtod(buf, NULL) != insn->val.f) {
+						IR_ASSERT(0 && "can't format float");
+					}
+				}
+				fprintf(f, "%s", buf);
 			}
 			break;
 		default:
@@ -1556,7 +1575,11 @@ int ir_mem_flush(void *ptr, size_t size)
 #else
 void *ir_mem_mmap(size_t size)
 {
-	void *ret = mmap(NULL, size, PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        int prot_flags = PROT_EXEC;
+#if defined(__NetBSD__)
+	prot_flags |= PROT_MPROTECT(PROT_READ|PROT_WRITE);
+#endif
+	void *ret = mmap(NULL, size, prot_flags, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 	if (ret == MAP_FAILED) {
 		ret = NULL;
 	}
@@ -1596,6 +1619,9 @@ int ir_mem_flush(void *ptr, size_t size)
 #if ((defined(__GNUC__) && ZEND_GCC_VERSION >= 4003) || __has_builtin(__builtin___clear_cache))
 	__builtin___clear_cache((char*)(ptr), (char*)(ptr) + size);
 #endif
+#if defined(__APPLE__) && defined(__aarch64__)
+	sys_icache_invalidate(ptr, size);
+#endif
 #ifdef HAVE_VALGRIND
 	VALGRIND_DISCARD_TRANSLATIONS(ptr, size);
 #endif

ext/opcache/jit/ir/ir_emit.c

@@ -24,6 +24,10 @@
 # include <psapi.h>
 #endif
 
+#if defined(__linux__) || defined(__sun)
+# include <alloca.h>
+#endif
+
 #define DASM_M_GROW(ctx, t, p, sz, need) \
   do { \
     size_t _sz = (sz), _need = (need); \
@@ -351,22 +355,21 @@ static void *ir_jmp_addr(ir_ctx *ctx, ir_insn *insn, ir_insn *addr_insn)
 	return addr;
 }
 
-static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref, uint8_t op_num)
+static int8_t ir_get_fused_reg(ir_ctx *ctx, ir_ref root, ir_ref ref_and_op)
 {
 	if (ctx->fused_regs) {
 		char key[10];
 		ir_ref val;
 
 		memcpy(key, &root, sizeof(ir_ref));
-		memcpy(key + 4, &ref, sizeof(ir_ref));
-		memcpy(key + 8, &op_num, sizeof(uint8_t));
+		memcpy(key + 4, &ref_and_op, sizeof(ir_ref));
 
-		val = ir_strtab_find(ctx->fused_regs, key, 9);
+		val = ir_strtab_find(ctx->fused_regs, key, 8);
 		if (val) {
 			return val;
 		}
 	}
-	return ctx->regs[ref][op_num];
+	return ((int8_t*)ctx->regs)[ref_and_op];
 }
 
 #if defined(__GNUC__)
@@ -393,6 +396,7 @@ static int ir_add_veneer(dasm_State *Dst, void *buffer, uint32_t ins, int *b, ui
 
 /* Forward Declarations */
 static void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block *bb);
+static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_reg, ir_reg tmp_fp_reg);
 static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb);
 
 typedef struct _ir_common_backend_data {
@@ -452,17 +456,200 @@ static IR_NEVER_INLINE void ir_emit_osr_entry_loads(ir_ctx *ctx, int b, ir_block
 			int32_t offset = -ir_binding_find(ctx, ref);
 
 			IR_ASSERT(offset > 0);
-			if (IR_IS_TYPE_INT(type)) {
-				ir_emit_load_mem_int(ctx, type, reg, ctx->spill_base, offset);
-			} else {
-				ir_emit_load_mem_fp(ctx, type, reg, ctx->spill_base, offset);
-			}
+			ir_emit_load_mem(ctx, type, reg, IR_MEM_BO(ctx->spill_base, offset));
 		} else {
 			IR_ASSERT(ctx->live_intervals[ctx->vregs[ref]]->flags & IR_LIVE_INTERVAL_SPILL_SPECIAL);
 		}
 	}
 }
 
+/*
+ * Parallel copy sequentialization algorithm
+ *
+ * The implementation is based on algorithm 1 desriebed in
+ * "Revisiting Out-of-SSA Translation for Correctness, Code Quality and Efficiency",
+ * Benoit Boissinot, Alain Darte, Fabrice Rastello, Benoit Dupont de Dinechin, Christophe Guillon.
+ * 2009 International Symposium on Code Generation and Optimization, Seattle, WA, USA, 2009,
+ * pp. 114-125, doi: 10.1109/CGO.2009.19.
+ */
+static int ir_parallel_copy(ir_ctx *ctx, ir_copy *copies, int count, ir_reg tmp_reg, ir_reg tmp_fp_reg)
+{
+	int i;
+	int8_t *pred, *loc, *types;
+	ir_reg to, from;
+	ir_type type;
+	ir_regset todo, ready, srcs;
+	ir_reg last_reg, last_fp_reg;
+
+	if (count == 1) {
+		to = copies[0].to;
+		from = copies[0].from;
+		IR_ASSERT(from != to);
+		type = copies[0].type;
+		if (IR_IS_TYPE_INT(type)) {
+			ir_emit_mov(ctx, type, to, from);
+		} else {
+			ir_emit_fp_mov(ctx, type, to, from);
+		}
+		return 1;
+	}
+
+	loc = alloca(IR_REG_NUM * 3 * sizeof(int8_t));
+	pred = loc + IR_REG_NUM;
+	types = pred + IR_REG_NUM;
+	todo = IR_REGSET_EMPTY;
+	srcs = IR_REGSET_EMPTY;
+
+	for (i = 0; i < count; i++) {
+		from = copies[i].from;
+		to = copies[i].to;
+		IR_ASSERT(from != to);
+		IR_REGSET_INCL(srcs, from);
+		loc[from] = from;
+		pred[to] = from;
+		types[from] = copies[i].type;
+		IR_ASSERT(!IR_REGSET_IN(todo, to));
+		IR_REGSET_INCL(todo, to);
+	}
+
+	ready = IR_REGSET_DIFFERENCE(todo, srcs);
+
+	if (ready == todo) {
+		for (i = 0; i < count; i++) {
+			from = copies[i].from;
+			to = copies[i].to;
+			IR_ASSERT(from != to);
+			type = copies[i].type;
+			if (IR_IS_TYPE_INT(type)) {
+				ir_emit_mov(ctx, type, to, from);
+			} else {
+				ir_emit_fp_mov(ctx, type, to, from);
+			}
+		}
+		return 1;
+	}
+
+	while (ready != IR_REGSET_EMPTY) {
+		ir_reg r;
+
+		to = ir_regset_pop_first(&ready);
+		from = pred[to];
+		r = loc[from];
+		type = types[from];
+		if (IR_IS_TYPE_INT(type)) {
+			ir_emit_mov_ext(ctx, type, to, r);
+		} else {
+			ir_emit_fp_mov(ctx, type, to, r);
+		}
+		IR_REGSET_EXCL(todo, to);
+		loc[from] = to;
+		if (from == r && IR_REGSET_IN(todo, from)) {
+			IR_REGSET_INCL(ready, from);
+		}
+	}
+	if (todo == IR_REGSET_EMPTY) {
+		return 1;
+	}
+
+	/* temporary registers may be the same as some of the destinations */
+	last_reg = IR_REG_NONE;
+	if (tmp_reg != IR_REG_NONE) {
+		IR_ASSERT(!IR_REGSET_IN(srcs, tmp_reg));
+		if (IR_REGSET_IN(todo, tmp_reg)) {
+			last_reg = tmp_reg;
+			IR_REGSET_EXCL(todo, tmp_reg);
+		}
+	}
+
+	last_fp_reg = IR_REG_NONE;
+	if (tmp_fp_reg != IR_REG_NONE) {
+		IR_ASSERT(!IR_REGSET_IN(srcs, tmp_fp_reg));
+		if (IR_REGSET_IN(todo, tmp_fp_reg)) {
+			last_fp_reg = tmp_fp_reg;
+			IR_REGSET_EXCL(todo, tmp_fp_reg);
+		}
+	}
+
+	while (todo != IR_REGSET_EMPTY) {
+		to = ir_regset_pop_first(&todo);
+		from = pred[to];
+		IR_ASSERT(to != loc[from]);
+		type = types[from];
+		if (IR_IS_TYPE_INT(type)) {
+#ifdef IR_HAVE_SWAP_INT
+			if (pred[from] == to) {
+				ir_emit_swap(ctx, type, to, from);
+				IR_REGSET_EXCL(todo, from);
+				loc[to] = from;
+				loc[from] = to;
+				continue;
+			}
+#endif
+			IR_ASSERT(tmp_reg != IR_REG_NONE);
+			IR_ASSERT(tmp_reg >= IR_REG_GP_FIRST && tmp_reg <= IR_REG_GP_LAST);
+			ir_emit_mov(ctx, type, tmp_reg, to);
+			loc[to] = tmp_reg;
+		} else {
+#ifdef IR_HAVE_SWAP_FP
+			if (pred[from] == to) {
+				ir_emit_swap_fp(ctx, type, to, from);
+				IR_REGSET_EXCL(todo, from);
+				loc[to] = from;
+				loc[from] = to;
+				continue;
+			}
+#endif
+			IR_ASSERT(tmp_fp_reg != IR_REG_NONE);
+			IR_ASSERT(tmp_fp_reg >= IR_REG_FP_FIRST && tmp_fp_reg <= IR_REG_FP_LAST);
+			ir_emit_fp_mov(ctx, type, tmp_fp_reg, to);
+			loc[to] = tmp_fp_reg;
+		}
+		while (1) {
+			ir_reg r;
+
+			from = pred[to];
+			r = loc[from];
+			type = types[from];
+			if (IR_IS_TYPE_INT(type)) {
+				ir_emit_mov_ext(ctx, type, to, r);
+			} else {
+				ir_emit_fp_mov(ctx, type, to, r);
+			}
+			IR_REGSET_EXCL(todo, to);
+			loc[from] = to;
+			if (from == r && IR_REGSET_IN(todo, from)) {
+				to = from;
+			} else {
+				break;
+			}
+		}
+	}
+
+	if (last_reg != IR_REG_NONE) {
+		to = last_reg;
+		from = pred[to];
+		type = types[from];
+		from = loc[from];
+		if (to != from) {
+			IR_ASSERT(IR_IS_TYPE_INT(type));
+			ir_emit_mov_ext(ctx, type, to, from);
+		}
+	}
+
+	if (last_fp_reg != IR_REG_NONE) {
+		to = last_fp_reg;
+		from = pred[to];
+		type = types[from];
+		from = loc[from];
+		if (to != from) {
+			IR_ASSERT(!IR_IS_TYPE_INT(type));
+			ir_emit_fp_mov(ctx, type, to, from);
+		}
+	}
+
+	return 1;
+}
+
 static void ir_emit_dessa_moves(ir_ctx *ctx, int b, ir_block *bb)
 {
 	uint32_t succ, k, n = 0, n2 = 0;

ext/opcache/jit/ir/ir_ra.c

@@ -1596,48 +1596,41 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2)
 	//ir_mem_free(ival);
 }
 
-static bool ir_try_coalesce(ir_ctx *ctx, ir_ref from, ir_ref to)
+static void ir_vregs_coalesce(ir_ctx *ctx, uint32_t v1, uint32_t v2, ir_ref from, ir_ref to)
 {
 	ir_ref i;
-	uint32_t v1 = ctx->vregs[from];
-	uint32_t v2 = ctx->vregs[to];
+	uint16_t f1 = ctx->live_intervals[v1]->flags;
+	uint16_t f2 = ctx->live_intervals[v2]->flags;
 
-	if (v1 != v2 && !ir_vregs_overlap(ctx, v1, v2)) {
-		uint16_t f1 = ctx->live_intervals[v1]->flags;
-		uint16_t f2 = ctx->live_intervals[v2]->flags;
-
-		if ((f1 & IR_LIVE_INTERVAL_COALESCED) && !(f2 & IR_LIVE_INTERVAL_COALESCED)) {
-			ir_vregs_join(ctx, v1, v2);
-			ctx->vregs[to] = v1;
-		} else if ((f2 & IR_LIVE_INTERVAL_COALESCED) && !(f1 & IR_LIVE_INTERVAL_COALESCED)) {
-			ir_vregs_join(ctx, v2, v1);
-			ctx->vregs[from] = v2;
-		} else if (from < to) {
-			ir_vregs_join(ctx, v1, v2);
-			if (f2 & IR_LIVE_INTERVAL_COALESCED) {
-				for (i = 1; i < ctx->insns_count; i++) {
-					if (ctx->vregs[i] == v2) {
-						ctx->vregs[i] = v1;
-					}
+	if ((f1 & IR_LIVE_INTERVAL_COALESCED) && !(f2 & IR_LIVE_INTERVAL_COALESCED)) {
+		ir_vregs_join(ctx, v1, v2);
+		ctx->vregs[to] = v1;
+	} else if ((f2 & IR_LIVE_INTERVAL_COALESCED) && !(f1 & IR_LIVE_INTERVAL_COALESCED)) {
+		ir_vregs_join(ctx, v2, v1);
+		ctx->vregs[from] = v2;
+	} else if (from < to) {
+		ir_vregs_join(ctx, v1, v2);
+		if (f2 & IR_LIVE_INTERVAL_COALESCED) {
+			for (i = 1; i < ctx->insns_count; i++) {
+				if (ctx->vregs[i] == v2) {
+					ctx->vregs[i] = v1;
 				}
-			} else {
-				ctx->vregs[to] = v1;
 			}
 		} else {
-			ir_vregs_join(ctx, v2, v1);
-			if (f1 & IR_LIVE_INTERVAL_COALESCED) {
-				for (i = 1; i < ctx->insns_count; i++) {
-					if (ctx->vregs[i] == v1) {
-						ctx->vregs[i] = v2;
-					}
-				}
-			} else {
-				ctx->vregs[from] = v2;
-			}
+			ctx->vregs[to] = v1;
+		}
+	} else {
+		ir_vregs_join(ctx, v2, v1);
+		if (f1 & IR_LIVE_INTERVAL_COALESCED) {
+			for (i = 1; i < ctx->insns_count; i++) {
+				if (ctx->vregs[i] == v1) {
+					ctx->vregs[i] = v2;
+				}
+			}
+		} else {
+			ctx->vregs[from] = v2;
 		}
-		return 1;
 	}
-	return 0;
 }
 
 static void ir_add_phi_move(ir_ctx *ctx, uint32_t b, ir_ref from, ir_ref to)
@@ -1879,10 +1872,40 @@ int ir_coalesce(ir_ctx *ctx)
 			if (insn->op == IR_PHI) {
 				input = ir_insn_op(insn, k);
 				if (input > 0) {
-					if (!ir_try_coalesce(ctx, input, use)) {
-						ir_add_phi_move(ctx, b, input, use);
+					uint32_t v1 = ctx->vregs[input];
+					uint32_t v2 = ctx->vregs[use];
+
+					if (v1 == v2) {
+						/* already coalesced */
 					} else {
-						compact = 1;
+						if (!ir_vregs_overlap(ctx, v1, v2)) {
+							ir_vregs_coalesce(ctx, v1, v2, input, use);
+							compact = 1;
+						} else {
+#if 1
+							ir_insn *input_insn = &ctx->ir_base[input];
+
+							if ((ir_op_flags[input_insn->op] & IR_OP_FLAG_COMMUTATIVE)
+							 && input_insn->op2 == use
+							 && input_insn->op1 != use
+							 && (ctx->live_intervals[v1]->use_pos->flags & IR_DEF_REUSES_OP1_REG)
+							 && ctx->live_intervals[v2]->end == IR_USE_LIVE_POS_FROM_REF(input)) {
+								ir_live_range *r = &ctx->live_intervals[v2]->range;
+
+								while (r->next) {
+									r = r->next;
+								}
+								r->end = IR_LOAD_LIVE_POS_FROM_REF(input);
+								ctx->live_intervals[v2]->end = IR_LOAD_LIVE_POS_FROM_REF(input);
+								ir_swap_operands(ctx, input, input_insn);
+								IR_ASSERT(!ir_vregs_overlap(ctx, v1, v2));
+								ir_vregs_coalesce(ctx, v1, v2, input, use);
+								compact = 1;
+								continue;
+							}
+#endif
+							ir_add_phi_move(ctx, b, input, use);
+						}
 					}
 				} else {
 					/* Move for constant input */